diff --git a/gui/CMakeLists.txt b/gui/CMakeLists.txt index 352af9a..fcdb4b3 100644 --- a/gui/CMakeLists.txt +++ b/gui/CMakeLists.txt @@ -5,7 +5,7 @@ set(CMAKE_INCLUDE_CURRENT_DIR ON) set(CMAKE_AUTOMOC ON) set(CMAKE_AUTOUIC ON) set(CMAKE_AUTORCC ON) -find_package(Qt5 REQUIRED COMPONENTS Core Widgets Gui Multimedia) +find_package(Qt5 REQUIRED COMPONENTS Core Widgets Gui Multimedia OpenGL) if(CHIAKI_GUI_ENABLE_QT_GAMEPAD) find_package(Qt5 REQUIRED COMPONENTS Gamepad) endif() @@ -29,12 +29,14 @@ add_executable(chiaki include/discoverymanager.h src/discoverymanager.cpp include/streamsession.h - src/streamsession.cpp) + src/streamsession.cpp + include/avopenglwidget.h + src/avopenglwidget.cpp) target_include_directories(chiaki PRIVATE include) target_link_libraries(chiaki chiaki-lib) target_link_libraries(chiaki FFMPEG::avcodec) -target_link_libraries(chiaki Qt5::Core Qt5::Widgets Qt5::Gui Qt5::Multimedia) +target_link_libraries(chiaki Qt5::Core Qt5::Widgets Qt5::Gui Qt5::Multimedia Qt5::OpenGL) if(CHIAKI_GUI_ENABLE_QT_GAMEPAD) target_link_libraries(chiaki Qt5::Gamepad) target_compile_definitions(chiaki PRIVATE CHIAKI_GUI_ENABLE_QT_GAMEPAD) diff --git a/gui/include/avopenglwidget.h b/gui/include/avopenglwidget.h new file mode 100644 index 0000000..1b3cccb --- /dev/null +++ b/gui/include/avopenglwidget.h @@ -0,0 +1,54 @@ +/* + * This file is part of Chiaki. + * + * Chiaki is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Chiaki is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Chiaki. If not, see . + */ + +#ifndef CHIAKI_AVOPENGLWIDGET_H +#define CHIAKI_AVOPENGLWIDGET_H + +#include + +extern "C" +{ +#include +} + +class VideoDecoder; + +class AVOpenGLWidget: public QOpenGLWidget +{ + Q_OBJECT + + private: + VideoDecoder *decoder; + + GLuint program; + GLuint vbo; + GLuint vao; + GLuint tex[3]; + unsigned int frame_width, frame_height; + + void UpdateTextures(AVFrame *frame); + + public: + explicit AVOpenGLWidget(VideoDecoder *decoder, QWidget *parent = nullptr); + + protected: + void initializeGL() override; + void resizeGL(int w, int h) override; + void paintGL() override; +}; + +#endif // CHIAKI_AVOPENGLWIDGET_H diff --git a/gui/include/streamwindow.h b/gui/include/streamwindow.h index dc68394..45d1e1b 100644 --- a/gui/include/streamwindow.h +++ b/gui/include/streamwindow.h @@ -23,6 +23,7 @@ #include "streamsession.h" class QLabel; +class AVOpenGLWidget; class StreamWindow: public QMainWindow { @@ -30,14 +31,12 @@ class StreamWindow: public QMainWindow public: explicit StreamWindow(const StreamSessionConnectInfo &connect_info, QWidget *parent = nullptr); - ~StreamWindow(); + ~StreamWindow() override; private: StreamSession *session; - QLabel *imageLabel; - - void SetImage(const QImage &image); + AVOpenGLWidget *av_widget; protected: void keyPressEvent(QKeyEvent *event) override; @@ -45,7 +44,6 @@ class StreamWindow: public QMainWindow void closeEvent(QCloseEvent *event) override; private slots: - void FramesAvailable(); void SessionQuit(ChiakiQuitReason reason, const QString &reason_str); }; diff --git a/gui/include/videodecoder.h b/gui/include/videodecoder.h index 7eb7630..29e0e01 100644 --- a/gui/include/videodecoder.h +++ b/gui/include/videodecoder.h @@ -24,7 +24,7 @@ extern "C" { #include -}; +} #include @@ -36,8 +36,8 @@ class VideoDecoder: public QObject VideoDecoder(); ~VideoDecoder(); - void PutFrame(uint8_t *buf, size_t buf_size); - QImage PullFrame(); + void PushFrame(uint8_t *buf, size_t buf_size); + AVFrame *PullFrame(); signals: void FramesAvailable(); diff --git a/gui/src/avopenglwidget.cpp b/gui/src/avopenglwidget.cpp new file mode 100644 index 0000000..938b2a0 --- /dev/null +++ b/gui/src/avopenglwidget.cpp @@ -0,0 +1,242 @@ +/* + * This file is part of Chiaki. + * + * Chiaki is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Chiaki is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Chiaki. If not, see . + */ + +#include +#include + +#include +#include +#include + +//#define DEBUG_OPENGL + +static const char *shader_vert_glsl = R"glsl( +#version 150 core + +in vec2 pos_attr; + +out vec2 uv_var; + +void main() +{ + uv_var = pos_attr; + gl_Position = vec4(pos_attr * vec2(2.0, -2.0) + vec2(-1.0, 1.0), 0.0, 1.0); +} +)glsl"; + +static const char *shader_frag_glsl = R"glsl( +#version 150 core + +uniform sampler2D tex_y; +uniform sampler2D tex_u; +uniform sampler2D tex_v; + +in vec2 uv_var; + +out vec4 out_color; + +void main() +{ + vec3 yuv = vec3( + texture2D(tex_y, uv_var).r, + texture2D(tex_u, uv_var).r - 0.5, + texture2D(tex_v, uv_var).r - 0.5); + vec3 rgb = mat3( + 1.0, 1.0, 1.0, + 0.0, -0.39393, 2.02839, + 1.14025, -0.58081, 0.0) * yuv; + out_color = vec4(rgb, 1.0); +} +)glsl"; + +static const float vert_pos[] = { + 0.0f, 0.0f, + 0.0f, 1.0f, + 1.0f, 0.0f, + 1.0f, 1.0f +}; + +AVOpenGLWidget::AVOpenGLWidget(VideoDecoder *decoder, QWidget *parent) + : QOpenGLWidget(parent), + decoder(decoder) +{ + QSurfaceFormat format; + format.setDepthBufferSize(0); + format.setStencilBufferSize(0); + format.setVersion(3, 2); + format.setProfile(QSurfaceFormat::CoreProfile); +#ifdef DEBUG_OPENGL + format.setOption(QSurfaceFormat::DebugContext, true); +#endif + setFormat(format); + + frame_width = 0; + frame_height = 0; + + connect(decoder, SIGNAL(FramesAvailable()), this, SLOT(update())); +} + +void AVOpenGLWidget::initializeGL() +{ + auto f = QOpenGLContext::currentContext()->extraFunctions(); + +#ifdef DEBUG_OPENGL + auto logger = new QOpenGLDebugLogger(this); + logger->initialize(); + connect(logger, &QOpenGLDebugLogger::messageLogged, this, [](const QOpenGLDebugMessage &msg) { + qDebug() << msg; + }); + logger->startLogging(); +#endif + + GLuint shader_vert = f->glCreateShader(GL_VERTEX_SHADER); + f->glShaderSource(shader_vert, 1, &shader_vert_glsl, nullptr); + f->glCompileShader(shader_vert); + + GLuint shader_frag = f->glCreateShader(GL_FRAGMENT_SHADER); + f->glShaderSource(shader_frag, 1, &shader_frag_glsl, nullptr); + f->glCompileShader(shader_frag); + + program = f->glCreateProgram(); + f->glAttachShader(program, shader_vert); + f->glAttachShader(program, shader_frag); + f->glBindAttribLocation(program, 0, "pos_attr"); + f->glLinkProgram(program); + + GLint linked = 0; + f->glGetProgramiv(program, GL_LINK_STATUS, &linked); + if(linked != GL_TRUE) + { + GLint info_log_size = 0; + f->glGetProgramiv(program, GL_INFO_LOG_LENGTH, &info_log_size); + std::vector info_log(info_log_size); + f->glGetProgramInfoLog(program, info_log_size, &info_log_size, info_log.data()); + f->glDeleteProgram(program); + // TODO: log to somewhere else + printf("Failed to Link Shader Program:\n%s\n", info_log.data()); + return; + } + + f->glGenTextures(3, tex); + uint8_t uv_default = 127; + for(int i=0; i<3; i++) + { + f->glBindTexture(GL_TEXTURE_2D, tex[i]); + f->glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + f->glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + f->glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + f->glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + f->glTexImage2D(GL_TEXTURE_2D, 0, GL_R8, 1, 1, 0, GL_RED, GL_UNSIGNED_BYTE, i > 0 ? &uv_default : nullptr); + } + + f->glUseProgram(program); + f->glUniform1i(f->glGetUniformLocation(program, "tex_y"), 0); + f->glUniform1i(f->glGetUniformLocation(program, "tex_u"), 1); + f->glUniform1i(f->glGetUniformLocation(program, "tex_v"), 2); + + f->glGenVertexArrays(1, &vao); + f->glBindVertexArray(vao); + + f->glGenBuffers(1, &vbo); + f->glBindBuffer(GL_ARRAY_BUFFER, vbo); + f->glBufferData(GL_ARRAY_BUFFER, sizeof(vert_pos), vert_pos, GL_STATIC_DRAW); + + f->glBindBuffer(GL_ARRAY_BUFFER, vbo); + f->glVertexAttribPointer(0, 2, GL_FLOAT, GL_FALSE, 0, nullptr); + f->glEnableVertexAttribArray(0); + + f->glCullFace(GL_BACK); + f->glEnable(GL_CULL_FACE); + f->glClearColor(0.0, 0.0, 0.0, 1.0); +} + +void AVOpenGLWidget::resizeGL(int w, int h) +{ +} + +void AVOpenGLWidget::UpdateTextures(AVFrame *frame) +{ + auto f = QOpenGLContext::currentContext()->functions(); + + if(frame->format != AV_PIX_FMT_YUV420P) + { + // TODO: log to somewhere else + printf("Invalid Format\n"); + return; + } + + frame_width = frame->width; + frame_height = frame->height; + + for(int i=0; i<3; i++) + { + f->glBindTexture(GL_TEXTURE_2D, tex[i]); + int width = frame->width; + int height = frame->height; + if(i > 0) + { + width /= 2; + height /= 2; + } + f->glTexImage2D(GL_TEXTURE_2D, 0, GL_R8, width, height, 0, GL_RED, GL_UNSIGNED_BYTE, frame->data[i]); + } +} + +void AVOpenGLWidget::paintGL() +{ + auto f = QOpenGLContext::currentContext()->extraFunctions(); + + f->glClear(GL_COLOR_BUFFER_BIT); + + AVFrame *next_frame = decoder->PullFrame(); + if(next_frame) + { + UpdateTextures(next_frame); + av_frame_free(&next_frame); + } + + GLsizei vp_width, vp_height; + if(!frame_width || !frame_height) + { + vp_width = width(); + vp_height = height(); + } + else + { + float aspect = (float)frame_width / (float)frame_height; + if(aspect < (float)width() / (float)height()) + { + vp_height = height(); + vp_width = (GLsizei)(vp_height * aspect); + } + else + { + vp_width = width(); + vp_height = (GLsizei)(vp_width / aspect); + } + } + + f->glViewport((width() - vp_width) / 2, (height() - vp_height) / 2, vp_width, vp_height); + + for(int i=0; i<3; i++) + { + f->glActiveTexture(GL_TEXTURE0 + i); + f->glBindTexture(GL_TEXTURE_2D, tex[i]); + } + + f->glDrawArrays(GL_TRIANGLE_STRIP, 0, 4); +} diff --git a/gui/src/main.cpp b/gui/src/main.cpp index 686820c..1c7f40f 100644 --- a/gui/src/main.cpp +++ b/gui/src/main.cpp @@ -71,8 +71,8 @@ int main(int argc, char *argv[]) connect_info.did = parser.value(did_option); chiaki_connect_video_profile_preset(&connect_info.video_profile, - CHIAKI_VIDEO_RESOLUTION_PRESET_360p, - CHIAKI_VIDEO_FPS_PRESET_30); + CHIAKI_VIDEO_RESOLUTION_PRESET_720p, + CHIAKI_VIDEO_FPS_PRESET_60); if(connect_info.registkey.isEmpty() || connect_info.ostype.isEmpty() || connect_info.auth.isEmpty() || connect_info.morning.isEmpty() || connect_info.did.isEmpty()) parser.showHelp(1); diff --git a/gui/src/streamsession.cpp b/gui/src/streamsession.cpp index 3ac90bd..f88126a 100644 --- a/gui/src/streamsession.cpp +++ b/gui/src/streamsession.cpp @@ -249,7 +249,7 @@ void StreamSession::PushAudioFrame(int16_t *buf, size_t samples_count) void StreamSession::PushVideoSample(uint8_t *buf, size_t buf_size) { - video_decoder.PutFrame(buf, buf_size); + video_decoder.PushFrame(buf, buf_size); } void StreamSession::Event(ChiakiEvent *event) diff --git a/gui/src/streamwindow.cpp b/gui/src/streamwindow.cpp index 65cc568..f51f576 100644 --- a/gui/src/streamwindow.cpp +++ b/gui/src/streamwindow.cpp @@ -17,6 +17,7 @@ #include #include +#include #include #include @@ -24,19 +25,12 @@ StreamWindow::StreamWindow(const StreamSessionConnectInfo &connect_info, QWidget *parent) : QMainWindow(parent) { - imageLabel = new QLabel(this); - setCentralWidget(imageLabel); - - imageLabel->setBackgroundRole(QPalette::Base); - imageLabel->setSizePolicy(QSizePolicy::Ignored, QSizePolicy::Ignored); - imageLabel->setScaledContents(true); - session = new StreamSession(connect_info, this); connect(session, &StreamSession::SessionQuit, this, &StreamWindow::SessionQuit); - connect(session->GetVideoDecoder(), &VideoDecoder::FramesAvailable, this, &StreamWindow::FramesAvailable); - FramesAvailable(); + av_widget = new AVOpenGLWidget(session->GetVideoDecoder(), this); + setCentralWidget(av_widget); grabKeyboard(); @@ -45,11 +39,8 @@ StreamWindow::StreamWindow(const StreamSessionConnectInfo &connect_info, QWidget StreamWindow::~StreamWindow() { -} - -void StreamWindow::SetImage(const QImage &image) -{ - imageLabel->setPixmap(QPixmap::fromImage(image)); + // make sure av_widget is always deleted before the session + delete av_widget; } void StreamWindow::keyPressEvent(QKeyEvent *event) @@ -67,22 +58,6 @@ void StreamWindow::closeEvent(QCloseEvent *) session->Stop(); } -void StreamWindow::FramesAvailable() -{ - QImage prev; - QImage image; - do - { - prev = image; - image = session->GetVideoDecoder()->PullFrame(); - } while(!image.isNull()); - - if(!prev.isNull()) - { - SetImage(prev); - } -} - void StreamWindow::SessionQuit(ChiakiQuitReason reason, const QString &reason_str) { if(reason == CHIAKI_QUIT_REASON_STOPPED) diff --git a/gui/src/videodecoder.cpp b/gui/src/videodecoder.cpp index e1fc80a..3afd532 100644 --- a/gui/src/videodecoder.cpp +++ b/gui/src/videodecoder.cpp @@ -39,7 +39,7 @@ VideoDecoder::~VideoDecoder() // TODO: free codec? } -void VideoDecoder::PutFrame(uint8_t *buf, size_t buf_size) +void VideoDecoder::PushFrame(uint8_t *buf, size_t buf_size) { { QMutexLocker locker(&mutex); @@ -54,48 +54,36 @@ void VideoDecoder::PutFrame(uint8_t *buf, size_t buf_size) emit FramesAvailable(); } -QImage VideoDecoder::PullFrame() +AVFrame *VideoDecoder::PullFrame() { QMutexLocker locker(&mutex); - AVFrame *frame = av_frame_alloc(); // TODO: handle !frame - int r = avcodec_receive_frame(codec_context, frame); - - if(r != 0) + // always try to pull as much as possible and return only the very last frame + AVFrame *frame_last = nullptr; + AVFrame *frame = nullptr; + while(true) { - if(r != AVERROR(EAGAIN)) - printf("decoding with ffmpeg failed!!\n"); - av_frame_free(&frame); - return QImage(); - } - - switch(frame->format) - { - case AV_PIX_FMT_YUV420P: - break; - default: - printf("unknown format %d\n", frame->format); - av_frame_free(&frame); - return QImage(); - } - - - QImage image(frame->width, frame->height, QImage::Format_RGB32); - for(int y=0; yheight; y++) - { - for(int x=0; xwidth; x++) + AVFrame *next_frame; + if(frame_last) { - int Y = frame->data[0][y * frame->linesize[0] + x] - 16; - int U = frame->data[1][(y/2) * frame->linesize[1] + (x/2)] - 128; - int V = frame->data[2][(y/2) * frame->linesize[2] + (x/2)] - 128; - int r = qBound(0, (298 * Y + 409 * V + 128) >> 8, 255); - int g = qBound(0, (298 * Y - 100 * U - 208 * V + 128) >> 8, 255); - int b = qBound(0, (298 * Y + 516 * U + 128) >> 8, 255); - image.setPixel(x, y, qRgb(r, g, b)); + av_frame_unref(frame_last); + next_frame = frame_last; + } + else + { + next_frame = av_frame_alloc(); + if(!next_frame) + return frame; + } + frame_last = frame; + frame = next_frame; + int r = avcodec_receive_frame(codec_context, frame); + if(r != 0) + { + if(r != AVERROR(EAGAIN)) + printf("decoding with ffmpeg failed!!\n"); // TODO: log somewhere else + av_frame_free(&frame); + return frame_last; } } - - av_frame_free(&frame); - - return image; -} \ No newline at end of file +}