github/chiaki
1
0
Fork 0
mirror of https://git.sr.ht/~thestr4ng3r/chiaki synced 2025-08-14 18:57:07 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions

Refactor Opus Decoding

Browse source
This commit is contained in:
Florian Märkl 2019-09-26 16:15:57 +02:00
commit 6748a94767
No known key found for this signature in database
GPG key ID: 125BC8A5A6A1E857
9 changed files with 240 additions and 82 deletions
Download patch file Download diff file Expand all files Collapse all files

2
gui/include/streamsession.h
View file

@ -19,6 +19,7 @@
#define CHIAKI_STREAMSESSION_H
#include <chiaki/session.h>
#include <chiaki/opusdecoder.h>
#include "videodecoder.h"
#include "exception.h"
@ -65,6 +66,7 @@ class StreamSession : public QObject
private:
SessionLog log;
ChiakiSession session;
ChiakiOpusDecoder opus_decoder;
#if CHIAKI_GUI_ENABLE_QT_GAMEPAD
QGamepad *gamepad;

9
gui/src/streamsession.cpp
View file

@ -64,6 +64,8 @@ StreamSession::StreamSession(const StreamSessionConnectInfo &connect_info, QObje
audio_output(nullptr),
audio_io(nullptr)
{
chiaki_opus_decoder_init(&opus_decoder, log.GetChiakiLog());
QByteArray host_str = connect_info.host.toUtf8();
ChiakiConnectInfo chiaki_connect_info;
@ -84,7 +86,11 @@ StreamSession::StreamSession(const StreamSessionConnectInfo &connect_info, QObje
if(err != CHIAKI_ERR_SUCCESS)
throw ChiakiException("Chiaki Session Init failed: " + QString::fromLocal8Bit(chiaki_error_string(err)));
chiaki_session_set_audio_cb(&session, AudioSettingsCb, AudioFrameCb, this);
chiaki_opus_decoder_set_cb(&opus_decoder, AudioSettingsCb, AudioFrameCb, this);
ChiakiAudioSink audio_sink;
chiaki_opus_decoder_get_sink(&opus_decoder, &audio_sink);
chiaki_session_set_audio_sink(&session, &audio_sink);
chiaki_session_set_video_sample_cb(&session, VideoSampleCb, this);
chiaki_session_set_event_cb(&session, EventCb, this);
@ -102,6 +108,7 @@ StreamSession::~StreamSession()
{
chiaki_session_join(&session);
chiaki_session_fini(&session);
chiaki_opus_decoder_fini(&opus_decoder);
#if CHIAKI_GUI_ENABLE_QT_GAMEPAD
delete gamepad;
#endif

10
lib/CMakeLists.txt
View file

@ -32,7 +32,8 @@ set(HEADER_FILES
include/chiaki/takionsendbuffer.h
include/chiaki/time.h
include/chiaki/fec.h
include/chiaki/regist.h)
include/chiaki/regist.h
include/chiaki/opusdecoder.h)
set(SOURCE_FILES
src/common.c
@ -67,7 +68,8 @@ set(SOURCE_FILES
src/takionsendbuffer.c
src/time.c
src/fec
src/regist.c)
src/regist.c
src/opusdecoder.c)
add_subdirectory(protobuf)
include_directories("${NANOPB_SOURCE_DIR}")
@ -77,10 +79,12 @@ include_directories("${CHIAKI_LIB_PROTO_INCLUDE_DIR}")
if(CHIAKI_LIB_ENABLE_OPUS)
find_package(Opus REQUIRED)
include_directories(${Opus_INCLUDE_DIRS})
add_definitions(-DCHIAKI_LIB_ENABLE_OPUS)
endif()
add_library(chiaki-lib ${HEADER_FILES} ${SOURCE_FILES} ${CHIAKI_LIB_PROTO_SOURCE_FILES} ${CHIAKI_LIB_PROTO_HEADER_FILES})
configure_file(config.h.in include/chiaki/config.h)
target_include_directories(chiaki-lib PUBLIC "${CMAKE_CURRENT_BINARY_DIR}/include")
add_dependencies(chiaki-lib chiaki-pb)
set_target_properties(chiaki-lib PROPERTIES OUTPUT_NAME chiaki)

23
lib/config.h.in Normal file
View file

@ -0,0 +1,23 @@
/*
* This file is part of Chiaki.
*
* Chiaki is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Chiaki is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Chiaki. If not, see <https://www.gnu.org/licenses/>.
*/
#ifndef CHIAKI_CONFIG_H
#define CHIAKI_CONFIG_H
#cmakedefine01 CHIAKI_LIB_ENABLE_OPUS
#endif // CHIAKI_CONFIG_H

16
lib/include/chiaki/audioreceiver.h
View file

@ -28,18 +28,26 @@
extern "C" {
#endif
typedef void (*ChiakiAudioSinkHeader)(ChiakiAudioHeader *header, void *user);
typedef void (*ChiakiAudioSinkFrame)(uint8_t *buf, size_t buf_size, void *user);
/**
* Sink that receives Audio encoded as Opus
*/
typedef struct chiaki_audio_sink_t
{
void *user;
ChiakiAudioSinkHeader header_cb;
ChiakiAudioSinkFrame frame_cb;
} ChiakiAudioSink;
typedef struct chiaki_audio_receiver_t
{
struct chiaki_session_t *session;
ChiakiLog *log;
ChiakiMutex mutex;
struct OpusDecoder *opus_decoder;
ChiakiAudioHeader audio_header;
ChiakiSeqNum16 frame_index_prev;
bool frame_index_startup; // whether frame_index_prev has definitely not wrapped yet
int16_t *pcm_buf;
size_t pcm_buf_size;
} ChiakiAudioReceiver;
CHIAKI_EXPORT ChiakiErrorCode chiaki_audio_receiver_init(ChiakiAudioReceiver *audio_receiver, struct chiaki_session_t *session);

63
lib/include/chiaki/opusdecoder.h Normal file
View file

@ -0,0 +1,63 @@
/*
* This file is part of Chiaki.
*
* Chiaki is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Chiaki is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Chiaki. If not, see <https://www.gnu.org/licenses/>.
*/
#ifndef CHIAKI_OPUSDECODER_H
#define CHIAKI_OPUSDECODER_H
#include <chiaki/config.h>
#if CHIAKI_LIB_ENABLE_OPUS
#include "audioreceiver.h"
#ifdef __cplusplus
extern "C" {
#endif
typedef void (*ChiakiOpusDecoderSettingsCallback)(uint32_t channels, uint32_t rate, void *user);
typedef void (*ChiakiOpusDecoderFrameCallback)(int16_t *buf, size_t samples_count, void *user);
typedef struct chiaki_opus_decoder_t
{
ChiakiLog *log;
struct OpusDecoder *opus_decoder;
ChiakiAudioHeader audio_header;
int16_t *pcm_buf;
size_t pcm_buf_size;
ChiakiOpusDecoderSettingsCallback settings_cb;
ChiakiOpusDecoderFrameCallback frame_cb;
void *cb_user;
} ChiakiOpusDecoder;
CHIAKI_EXPORT void chiaki_opus_decoder_init(ChiakiOpusDecoder *decoder, ChiakiLog *log);
CHIAKI_EXPORT void chiaki_opus_decoder_fini(ChiakiOpusDecoder *decoder);
CHIAKI_EXPORT void chiaki_opus_decoder_get_sink(ChiakiOpusDecoder *decoder, ChiakiAudioSink *sink);
static inline void chiaki_opus_decoder_set_cb(ChiakiOpusDecoder *decoder, ChiakiOpusDecoderSettingsCallback settings_cb, ChiakiOpusDecoderFrameCallback frame_cb, void *user)
{
decoder->settings_cb = settings_cb;
decoder->frame_cb = frame_cb;
decoder->cb_user = user;
}
#ifdef __cplusplus
}
#endif
#endif
#endif // CHIAKI_OPUSDECODER_H

21
lib/include/chiaki/session.h
View file

@ -129,8 +129,6 @@ typedef struct chiaki_event_t
} ChiakiEvent;
typedef void (*ChiakiEventCallback)(ChiakiEvent *event, void *user);
typedef void (*ChiakiAudioSettingsCallback)(uint32_t channels, uint32_t rate, void *user);
typedef void (*ChiakiAudioFrameCallback)(int16_t *buf, size_t samples_count, void *user);
/**
* buf will always have an allocated padding of at least CHIAKI_VIDEO_BUFFER_PADDING_SIZE after buf_size
@ -166,11 +164,9 @@ typedef struct chiaki_session_t
ChiakiEventCallback event_cb;
void *event_cb_user;
ChiakiAudioSettingsCallback audio_settings_cb;
ChiakiAudioFrameCallback audio_frame_cb;
void *audio_cb_user;
ChiakiVideoSampleCallback video_sample_cb;
void *video_sample_cb_user;
ChiakiAudioSink audio_sink;
ChiakiThread session_thread;
@ -210,19 +206,20 @@ static inline void chiaki_session_set_event_cb(ChiakiSession *session, ChiakiEve
session->event_cb_user = user;
}
static inline void chiaki_session_set_audio_cb(ChiakiSession *session, ChiakiAudioSettingsCallback settings_cb, ChiakiAudioFrameCallback frame_cb, void *user)
{
session->audio_settings_cb = settings_cb;
session->audio_frame_cb = frame_cb;
session->audio_cb_user = user;
}
static inline void chiaki_session_set_video_sample_cb(ChiakiSession *session, ChiakiVideoSampleCallback cb, void *user)
{
session->video_sample_cb = cb;
session->video_sample_cb_user = user;
}
/**
* @param sink contents are copied
*/
static inline void chiaki_session_set_audio_sink(ChiakiSession *session, ChiakiAudioSink *sink)
{
session->audio_sink = *sink;
}
#ifdef __cplusplus
}
#endif

66
lib/src/audioreceiver.c
View file

@ -18,10 +18,6 @@
#include <chiaki/audioreceiver.h>
#include <chiaki/session.h>
#ifdef CHIAKI_LIB_ENABLE_OPUS
#include <opus/opus.h>
#endif
#include <string.h>
static void chiaki_audio_receiver_frame(ChiakiAudioReceiver *audio_receiver, ChiakiSeqNum16 frame_index, uint8_t *buf, size_t buf_size);
@ -30,15 +26,10 @@ CHIAKI_EXPORT ChiakiErrorCode chiaki_audio_receiver_init(ChiakiAudioReceiver *au
{
audio_receiver->session = session;
audio_receiver->log = session->log;
audio_receiver->opus_decoder = NULL;
memset(&audio_receiver->audio_header, 0, sizeof(audio_receiver->audio_header));
audio_receiver->frame_index_prev = 0;
audio_receiver->frame_index_startup = true;
audio_receiver->pcm_buf = NULL;
audio_receiver->pcm_buf_size = 0;
ChiakiErrorCode err = chiaki_mutex_init(&audio_receiver->mutex, false);
if(err != CHIAKI_ERR_SUCCESS)
return err;
@ -53,7 +44,6 @@ CHIAKI_EXPORT void chiaki_audio_receiver_fini(ChiakiAudioReceiver *audio_receive
opus_decoder_destroy(audio_receiver->opus_decoder);
#endif
chiaki_mutex_fini(&audio_receiver->mutex);
free(audio_receiver->pcm_buf);
}
@ -67,46 +57,10 @@ CHIAKI_EXPORT void chiaki_audio_receiver_stream_info(ChiakiAudioReceiver *audio_
CHIAKI_LOGI(audio_receiver->log, " rate = %d", audio_header->rate);
CHIAKI_LOGI(audio_receiver->log, " frame size = %d", audio_header->frame_size);
CHIAKI_LOGI(audio_receiver->log, " unknown = %d", audio_header->unknown);
memcpy(&audio_receiver->audio_header, audio_header, sizeof(audio_receiver->audio_header));
#ifdef CHIAKI_LIB_ENABLE_OPUS
opus_decoder_destroy(audio_receiver->opus_decoder);
if(audio_receiver->session->audio_sink.header_cb)
audio_receiver->session->audio_sink.header_cb(audio_header, audio_receiver->session->audio_sink.user);
int error;
audio_receiver->opus_decoder = opus_decoder_create(audio_header->rate, audio_header->channels, &error);
if(error != OPUS_OK)
{
CHIAKI_LOGE(audio_receiver->log, "Audio Receiver failed to initialize opus decoder: %s", opus_strerror(error));
goto beach;
}
CHIAKI_LOGI(audio_receiver->log, "Audio Receiver initialized opus decoder with the settings above");
size_t pcm_buf_size_required = chiaki_audio_header_frame_buf_size(audio_header);
int16_t *pcm_buf_old = audio_receiver->pcm_buf;
if(!audio_receiver->pcm_buf || audio_receiver->pcm_buf_size != pcm_buf_size_required)
audio_receiver->pcm_buf = realloc(audio_receiver->pcm_buf, pcm_buf_size_required);
if(!audio_receiver->pcm_buf)
{
free(pcm_buf_old);
CHIAKI_LOGE(audio_receiver->log, "Audio Receiver failed to alloc pcm buffer");
opus_decoder_destroy(audio_receiver->opus_decoder);
audio_receiver->opus_decoder = NULL;
audio_receiver->pcm_buf_size = 0;
goto beach;
}
audio_receiver->pcm_buf_size = pcm_buf_size_required;
if(audio_receiver->session->audio_settings_cb)
audio_receiver->session->audio_settings_cb(audio_header->channels, audio_header->rate, audio_receiver->session->audio_cb_user);
#else
CHIAKI_LOGE(audio_receiver->log, "Opus disabled, not initializing decoder");
#endif
beach:
chiaki_mutex_unlock(&audio_receiver->mutex);
}
@ -168,27 +122,15 @@ CHIAKI_EXPORT void chiaki_audio_receiver_av_packet(ChiakiAudioReceiver *audio_re
static void chiaki_audio_receiver_frame(ChiakiAudioReceiver *audio_receiver, ChiakiSeqNum16 frame_index, uint8_t *buf, size_t buf_size)
{
#ifdef CHIAKI_LIB_ENABLE_OPUS
chiaki_mutex_lock(&audio_receiver->mutex);
if(!audio_receiver->opus_decoder)
{
CHIAKI_LOGE(audio_receiver->log, "Received audio frame, but opus decoder is not initialized");
goto beach;
}
if(!chiaki_seq_num_16_gt(frame_index, audio_receiver->frame_index_prev))
goto beach;
audio_receiver->frame_index_prev = frame_index;
int r = opus_decode(audio_receiver->opus_decoder, buf, (opus_int32)buf_size, audio_receiver->pcm_buf, audio_receiver->audio_header.frame_size, 0);
if(r < 1)
CHIAKI_LOGE(audio_receiver->log, "Decoding audio frame with opus failed: %s", opus_strerror(r));
else
audio_receiver->session->audio_frame_cb(audio_receiver->pcm_buf, (size_t)r, audio_receiver->session->audio_cb_user);
if(audio_receiver->session->audio_sink.frame_cb)
audio_receiver->session->audio_sink.frame_cb(buf, buf_size, audio_receiver->session->audio_sink.user);
beach:
chiaki_mutex_unlock(&audio_receiver->mutex);
#endif
}

112
lib/src/opusdecoder.c Normal file
View file

@ -0,0 +1,112 @@
/*
* This file is part of Chiaki.
*
* Chiaki is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Chiaki is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Chiaki. If not, see <https://www.gnu.org/licenses/>.
*/
#include <chiaki/config.h>
#if CHIAKI_LIB_ENABLE_OPUS
#include <chiaki/opusdecoder.h>
#include <opus/opus.h>
#include <string.h>
static void chiaki_opus_decoder_header(ChiakiAudioHeader *header, void *user);
static void chiaki_opus_decoder_frame(uint8_t *buf, size_t buf_size, void *user);
CHIAKI_EXPORT void chiaki_opus_decoder_init(ChiakiOpusDecoder *decoder, ChiakiLog *log)
{
decoder->log = log;
decoder->opus_decoder = NULL;
memset(&decoder->audio_header, 0, sizeof(decoder->audio_header));
decoder->pcm_buf = NULL;
decoder->pcm_buf_size = 0;
decoder->cb_user = NULL;
decoder->settings_cb = NULL;
decoder->frame_cb = NULL;
}
CHIAKI_EXPORT void chiaki_opus_decoder_fini(ChiakiOpusDecoder *decoder)
{
free(decoder->pcm_buf);
}
CHIAKI_EXPORT void chiaki_opus_decoder_get_sink(ChiakiOpusDecoder *decoder, ChiakiAudioSink *sink)
{
sink->user = decoder;
sink->header_cb = chiaki_opus_decoder_header;
sink->frame_cb = chiaki_opus_decoder_frame;
}
static void chiaki_opus_decoder_header(ChiakiAudioHeader *header, void *user)
{
ChiakiOpusDecoder *decoder = user;
memcpy(&decoder->audio_header, header, sizeof(decoder->audio_header));
opus_decoder_destroy(decoder->opus_decoder);
int error;
decoder->opus_decoder = opus_decoder_create(header->rate, header->channels, &error);
if(error != OPUS_OK)
{
CHIAKI_LOGE(decoder->log, "ChiakiOpusDecoder failed to initialize opus decoder: %s", opus_strerror(error));
decoder->opus_decoder = NULL;
return;
}
CHIAKI_LOGI(decoder->log, "ChiakiOpusDecoder initialized");
size_t pcm_buf_size_required = chiaki_audio_header_frame_buf_size(header);
int16_t *pcm_buf_old = decoder->pcm_buf;
if(!decoder->pcm_buf || decoder->pcm_buf_size != pcm_buf_size_required)
decoder->pcm_buf = realloc(decoder->pcm_buf, pcm_buf_size_required);
if(!decoder->pcm_buf)
{
free(pcm_buf_old);
CHIAKI_LOGE(decoder->log, "ChiakiOpusDecoder failed to alloc pcm buffer");
opus_decoder_destroy(decoder->opus_decoder);
decoder->opus_decoder = NULL;
decoder->pcm_buf_size = 0;
return;
}
decoder->pcm_buf_size = pcm_buf_size_required;
if(decoder->settings_cb)
decoder->settings_cb(header->channels, header->rate, decoder->cb_user);
}
static void chiaki_opus_decoder_frame(uint8_t *buf, size_t buf_size, void *user)
{
ChiakiOpusDecoder *decoder = user;
if(!decoder->opus_decoder)
{
CHIAKI_LOGE(decoder->log, "Received audio frame, but opus decoder is not initialized");
return;
}
int r = opus_decode(decoder->opus_decoder, buf, (opus_int32)buf_size, decoder->pcm_buf, decoder->audio_header.frame_size, 0);
if(r < 1)
CHIAKI_LOGE(decoder->log, "Decoding audio frame with opus failed: %s", opus_strerror(r));
else if(decoder->frame_cb)
decoder->frame_cb(decoder->pcm_buf, (size_t)r, decoder->cb_user);
}
#endif