From 55002a45856ddd9e2e281cde0c6cf59325d9f9f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20M=C3=A4rkl?= Date: Tue, 6 Aug 2019 12:41:55 +0200 Subject: [PATCH] Make Video Decoding more stable --- gui/CMakeLists.txt | 1 + gui/include/exception.h | 35 +++++++++++++++++++++++++ gui/include/streamsession.h | 9 +++---- gui/include/videodecoder.h | 8 ++++++ gui/src/videodecoder.cpp | 51 ++++++++++++++++++++++++++++++------ lib/include/chiaki/session.h | 4 +++ lib/include/chiaki/video.h | 5 ++++ lib/src/frameprocessor.c | 5 ++-- lib/src/streamconnection.c | 11 +++++++- lib/src/takion.c | 5 +++- 10 files changed, 116 insertions(+), 18 deletions(-) create mode 100644 gui/include/exception.h diff --git a/gui/CMakeLists.txt b/gui/CMakeLists.txt index f227622..d5f41ee 100644 --- a/gui/CMakeLists.txt +++ b/gui/CMakeLists.txt @@ -13,6 +13,7 @@ endif() find_package(FFMPEG REQUIRED COMPONENTS avcodec) add_executable(chiaki + include/exception.h src/main.cpp include/streamwindow.h src/streamwindow.cpp diff --git a/gui/include/exception.h b/gui/include/exception.h new file mode 100644 index 0000000..ebeea1d --- /dev/null +++ b/gui/include/exception.h @@ -0,0 +1,35 @@ +/* + * This file is part of Chiaki. + * + * Chiaki is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Chiaki is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Chiaki. If not, see . + */ + +#ifndef CHIAKI_EXCEPTION_H +#define CHIAKI_EXCEPTION_H + +#include + +#include + +class Exception : public std::exception +{ + private: + QString msg; + + public: + explicit Exception(const QString &msg) : msg(msg) {} + const char *what() const noexcept override { return msg.toLocal8Bit().constData(); } +}; + +#endif // CHIAKI_EXCEPTION_H diff --git a/gui/include/streamsession.h b/gui/include/streamsession.h index 7b9041c..710c446 100644 --- a/gui/include/streamsession.h +++ b/gui/include/streamsession.h @@ -19,6 +19,7 @@ #define CHIAKI_STREAMSESSION_H #include "videodecoder.h" +#include "exception.h" #include #include @@ -33,14 +34,10 @@ class QAudioOutput; class QIODevice; class QKeyEvent; -class ChiakiException : public std::exception +class ChiakiException: public Exception { - private: - QString msg; - public: - explicit ChiakiException(const QString &msg) : msg(msg) {} - const char *what() const noexcept override { return msg.toLocal8Bit().constData(); } + explicit ChiakiException(const QString &msg) : Exception(msg) {}; }; struct StreamSessionConnectInfo diff --git a/gui/include/videodecoder.h b/gui/include/videodecoder.h index 29e0e01..12316aa 100644 --- a/gui/include/videodecoder.h +++ b/gui/include/videodecoder.h @@ -18,6 +18,8 @@ #ifndef CHIAKI_VIDEODECODER_H #define CHIAKI_VIDEODECODER_H +#include "exception.h" + #include #include @@ -28,6 +30,12 @@ extern "C" #include +class VideoDecoderException: public Exception +{ + public: + explicit VideoDecoderException(const QString &msg) : Exception(msg) {}; +}; + class VideoDecoder: public QObject { Q_OBJECT diff --git a/gui/src/videodecoder.cpp b/gui/src/videodecoder.cpp index 3afd532..e78662b 100644 --- a/gui/src/videodecoder.cpp +++ b/gui/src/videodecoder.cpp @@ -23,20 +23,25 @@ VideoDecoder::VideoDecoder() { - codec = avcodec_find_decoder(AV_CODEC_ID_H264); // TODO: handle !codec - codec_context = avcodec_alloc_context3(codec); // TODO: handle !codec_context + codec = avcodec_find_decoder(AV_CODEC_ID_H264); + if(!codec) + throw VideoDecoderException("H264 Codec not available"); - if(codec->capabilities & AV_CODEC_CAP_TRUNCATED) - codec_context->flags |= AV_CODEC_FLAG_TRUNCATED; + codec_context = avcodec_alloc_context3(codec); + if(!codec_context) + throw VideoDecoderException("Failed to alloc codec context"); - avcodec_open2(codec_context, codec, nullptr); // TODO: handle < 0 + if(avcodec_open2(codec_context, codec, nullptr) < 0) + { + avcodec_free_context(&codec_context); + throw VideoDecoderException("Failed to open codec context"); + } } VideoDecoder::~VideoDecoder() { avcodec_close(codec_context); - avcodec_free_context(&codec_context); // TODO: does close by itself too? - // TODO: free codec? + avcodec_free_context(&codec_context); } void VideoDecoder::PushFrame(uint8_t *buf, size_t buf_size) @@ -48,12 +53,42 @@ void VideoDecoder::PushFrame(uint8_t *buf, size_t buf_size) av_init_packet(&packet); packet.data = buf; packet.size = buf_size; - avcodec_send_packet(codec_context, &packet); + int r; +send_packet: + r = avcodec_send_packet(codec_context, &packet); + if(r != 0) + { + if(r == AVERROR(EAGAIN)) + { + printf("avcodec internal buffer is full, removing frames\n"); // TODO: log to somewhere else + AVFrame *frame = av_frame_alloc(); + if(!frame) + { + printf("failed to alloc frame\n"); + return; + } + r = avcodec_receive_frame(codec_context, frame); + av_frame_free(&frame); + if(r != 0) + { + printf("failed to pull packet\n"); + return; + } + goto send_packet; + } + else + { + printf("failed to push frame\n"); + return; + } + } } emit FramesAvailable(); } +#include + AVFrame *VideoDecoder::PullFrame() { QMutexLocker locker(&mutex); diff --git a/lib/include/chiaki/session.h b/lib/include/chiaki/session.h index d2783b3..611b161 100644 --- a/lib/include/chiaki/session.h +++ b/lib/include/chiaki/session.h @@ -119,6 +119,10 @@ typedef struct chiaki_event_t typedef void (*ChiakiEventCallback)(ChiakiEvent *event, void *user); typedef void (*ChiakiAudioFrameCallback)(int16_t *buf, size_t samples_count, void *user); + +/** + * buf will always have an allocated padding of at least CHIAKI_VIDEO_BUFFER_PADDING_SIZE after buf_size + */ typedef void (*ChiakiVideoSampleCallback)(uint8_t *buf, size_t buf_size, void *user); diff --git a/lib/include/chiaki/video.h b/lib/include/chiaki/video.h index 76a4f61..5555815 100644 --- a/lib/include/chiaki/video.h +++ b/lib/include/chiaki/video.h @@ -33,6 +33,11 @@ typedef struct chiaki_video_profile_t uint8_t *header; } ChiakiVideoProfile; +/** + * Padding for FFMPEG + */ +#define CHIAKI_VIDEO_BUFFER_PADDING_SIZE 64 + #ifdef __cplusplus } #endif diff --git a/lib/src/frameprocessor.c b/lib/src/frameprocessor.c index 4b76c3c..2df4fc0 100644 --- a/lib/src/frameprocessor.c +++ b/lib/src/frameprocessor.c @@ -17,6 +17,7 @@ #include #include +#include #include @@ -117,7 +118,7 @@ CHIAKI_EXPORT ChiakiErrorCode chiaki_frame_processor_alloc_frame(ChiakiFrameProc if(frame_processor->frame_buf_size < frame_buf_size_required) { free(frame_processor->frame_buf); - frame_processor->frame_buf = malloc(frame_buf_size_required); + frame_processor->frame_buf = malloc(frame_buf_size_required + CHIAKI_VIDEO_BUFFER_PADDING_SIZE); if(!frame_processor->frame_buf) { frame_processor->frame_buf_size = 0; @@ -125,7 +126,7 @@ CHIAKI_EXPORT ChiakiErrorCode chiaki_frame_processor_alloc_frame(ChiakiFrameProc } frame_processor->frame_buf_size = frame_buf_size_required; } - memset(frame_processor->frame_buf, 0, frame_buf_size_required); + memset(frame_processor->frame_buf, 0, frame_buf_size_required + CHIAKI_VIDEO_BUFFER_PADDING_SIZE); return CHIAKI_ERR_SUCCESS; } diff --git a/lib/src/streamconnection.c b/lib/src/streamconnection.c index 8007f42..b1c17eb 100644 --- a/lib/src/streamconnection.c +++ b/lib/src/streamconnection.c @@ -539,6 +539,15 @@ static bool pb_decode_resolution(pb_istream_t *stream, const pb_field_t *field, return true; } + uint8_t *header_buf_padded = realloc(header_buf.buf, header_buf.size + CHIAKI_VIDEO_BUFFER_PADDING_SIZE); + if(!header_buf_padded) + { + free(header_buf.buf); + CHIAKI_LOGE(&ctx->stream_connection->session->log, "Failed to realloc video header with padding"); + return true; + } + memset(header_buf_padded + header_buf.size, 0, CHIAKI_VIDEO_BUFFER_PADDING_SIZE); + if(ctx->video_profiles_count >= CHIAKI_VIDEO_PROFILES_MAX) { CHIAKI_LOGE(&ctx->stream_connection->session->log, "Received more resolutions than the maximum"); @@ -549,7 +558,7 @@ static bool pb_decode_resolution(pb_istream_t *stream, const pb_field_t *field, profile->width = resolution.width; profile->height = resolution.height; profile->header_sz = header_buf.size; - profile->header = header_buf.buf; + profile->header = header_buf_padded; return true; } diff --git a/lib/src/takion.c b/lib/src/takion.c index 8c2e18e..47f7f97 100644 --- a/lib/src/takion.c +++ b/lib/src/takion.c @@ -808,8 +808,8 @@ static void takion_handle_packet(ChiakiTakion *takion, uint8_t *buf, size_t buf_ break; default: CHIAKI_LOGW(takion->log, "Takion packet with unknown type %#x received", base_type); + chiaki_log_hexdump(takion->log, CHIAKI_LOG_WARNING, buf, buf_size); free(buf); - //chiaki_log_hexdump(takion->log, CHIAKI_LOG_WARNING, buf, buf_size); break; } } @@ -868,7 +868,10 @@ static void takion_flush_data_queue(ChiakiTakion *takion) CHIAKI_LOGW(takion->log, "Takion received data with unexpected nonzero %#x at buf+6", zero_a); if(data_type != CHIAKI_TAKION_MESSAGE_DATA_TYPE_PROTOBUF && data_type != CHIAKI_TAKION_MESSAGE_DATA_TYPE_9) + { CHIAKI_LOGW(takion->log, "Takion received data with unexpected data type %#x", data_type); + chiaki_log_hexdump(takion->log, CHIAKI_LOG_WARNING, entry->packet_buf, entry->packet_size); + } else if(takion->cb) { ChiakiTakionEvent event = { 0 };