github/chiaki
1
0
Fork 0
mirror of https://git.sr.ht/~thestr4ng3r/chiaki synced 2025-08-21 14:03:11 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions

Implement Audio FEC

Browse source
This commit is contained in:
Florian Märkl 2019-07-31 22:42:07 +02:00
commit 1e8953f76c
No known key found for this signature in database
GPG key ID: 125BC8A5A6A1E857
4 changed files with 76 additions and 30 deletions
Download patch file Download diff file Expand all files Collapse all files

5
lib/include/chiaki/audioreceiver.h
View file

@ -21,6 +21,7 @@
#include "common.h"
#include "log.h"
#include "audio.h"
#include "takion.h"
#include "thread.h"
#ifdef __cplusplus
@ -35,12 +36,14 @@ typedef struct chiaki_audio_receiver_t
ChiakiMutex mutex;
struct OpusDecoder *opus_decoder;
ChiakiAudioHeader audio_header;
ChiakiSeqNum16 frame_index_prev;
bool frame_index_startup; // whether frame_index_prev has definitely not wrapped yet
} ChiakiAudioReceiver;
CHIAKI_EXPORT ChiakiErrorCode chiaki_audio_receiver_init(ChiakiAudioReceiver *audio_receiver, struct chiaki_session_t *session);
CHIAKI_EXPORT void chiaki_audio_receiver_fini(ChiakiAudioReceiver *audio_receiver);
CHIAKI_EXPORT void chiaki_audio_receiver_stream_info(ChiakiAudioReceiver *audio_receiver, ChiakiAudioHeader *audio_header);
CHIAKI_EXPORT void chiaki_audio_receiver_frame_packet(ChiakiAudioReceiver *audio_receiver, uint8_t *buf, size_t buf_size);
CHIAKI_EXPORT void chiaki_audio_receiver_av_packet(ChiakiAudioReceiver *audio_receiver, ChiakiTakionAVPacket *packet);
static inline ChiakiAudioReceiver *chiaki_audio_receiver_new(struct chiaki_session_t *session)
{

4
lib/include/chiaki/takion.h
View file

@ -63,6 +63,10 @@ typedef struct chiaki_takion_av_packet_t
size_t data_size;
} ChiakiTakionAVPacket;
static inline uint8_t chiaki_takion_av_packet_audio_unit_size(ChiakiTakionAVPacket *packet) { return packet->units_in_frame_fec >> 8; }
static inline uint8_t chiaki_takion_av_packet_audio_source_units_count(ChiakiTakionAVPacket *packet) { return packet->units_in_frame_fec & 0xf; }
static inline uint8_t chiaki_takion_av_packet_audio_fec_units_count(ChiakiTakionAVPacket *packet) { return (packet->units_in_frame_fec >> 4) & 0xf; }
typedef struct chiaki_takion_congestion_packet_t
{

70
lib/src/audioreceiver.c
View file

@ -22,6 +22,7 @@
#include <string.h>
static void chiaki_audio_receiver_frame(ChiakiAudioReceiver *audio_receiver, ChiakiSeqNum16 frame_index, uint8_t *buf, size_t buf_size);
CHIAKI_EXPORT ChiakiErrorCode chiaki_audio_receiver_init(ChiakiAudioReceiver *audio_receiver, ChiakiSession *session)
{
@ -30,6 +31,9 @@ CHIAKI_EXPORT ChiakiErrorCode chiaki_audio_receiver_init(ChiakiAudioReceiver *au
audio_receiver->opus_decoder = NULL;
memset(&audio_receiver->audio_header, 0, sizeof(audio_receiver->audio_header));
audio_receiver->frame_index_prev = 0;
audio_receiver->frame_index_startup = true;
ChiakiErrorCode err = chiaki_mutex_init(&audio_receiver->mutex, false);
if(err != CHIAKI_ERR_SUCCESS)
return err;
@ -70,18 +74,77 @@ CHIAKI_EXPORT void chiaki_audio_receiver_stream_info(ChiakiAudioReceiver *audio_
chiaki_mutex_unlock(&audio_receiver->mutex);
}
CHIAKI_EXPORT void chiaki_audio_receiver_av_packet(ChiakiAudioReceiver *audio_receiver, ChiakiTakionAVPacket *packet)
{
if(packet->codec != 5)
{
CHIAKI_LOGE(audio_receiver->log, "Received Audio Packet with unknown Codec");
return;
}
CHIAKI_EXPORT void chiaki_audio_receiver_frame_packet(ChiakiAudioReceiver *audio_receiver, uint8_t *buf, size_t buf_size)
// this is mostly observation-based, so may not necessarily cover everything yet.
uint8_t source_units_count = chiaki_takion_av_packet_audio_source_units_count(packet);
uint8_t fec_units_count = chiaki_takion_av_packet_audio_fec_units_count(packet);
uint8_t unit_size = chiaki_takion_av_packet_audio_unit_size(packet);
if(!packet->data_size)
{
CHIAKI_LOGE(audio_receiver->log, "Audio AV Packet is empty");
return;
}
if((uint16_t)fec_units_count + (uint16_t)source_units_count != packet->units_in_frame_total)
{
CHIAKI_LOGE(audio_receiver->log, "Source Units + FEC Units != Total Units in Audio AV Packet");
return;
}
if(packet->data_size != (size_t)unit_size * (size_t)packet->units_in_frame_total)
{
CHIAKI_LOGE(audio_receiver->log, "Audio AV Packet size mismatch");
return;
}
if(packet->frame_index > (1 << 15))
audio_receiver->frame_index_startup = false;
for(size_t i=0; i<source_units_count+fec_units_count; i++)
{
ChiakiSeqNum16 frame_index;
if(i < source_units_count)
frame_index = packet->frame_index + i;
else
{
// fec
size_t fec_index = i - source_units_count;
// first packets will contain the same frame multiple times, ignore those
if(audio_receiver->frame_index_startup && packet->frame_index + fec_index < fec_units_count + 1)
continue;
frame_index = packet->frame_index - fec_units_count + fec_index;
}
chiaki_audio_receiver_frame(audio_receiver->session->audio_receiver, frame_index, packet->data + unit_size * i, unit_size);
}
}
static void chiaki_audio_receiver_frame(ChiakiAudioReceiver *audio_receiver, ChiakiSeqNum16 frame_index, uint8_t *buf, size_t buf_size)
{
chiaki_mutex_lock(&audio_receiver->mutex);
if(!audio_receiver->opus_decoder)
{
CHIAKI_LOGE(audio_receiver->log, "Received audio frame, but opus decoder is not initialized");
chiaki_mutex_unlock(&audio_receiver->mutex);
return;
goto beach;
}
if(!chiaki_seq_num_16_gt(frame_index, audio_receiver->frame_index_prev))
goto beach;
audio_receiver->frame_index_prev = frame_index;
// TODO: don't malloc
opus_int16 *pcm = malloc(audio_receiver->audio_header.frame_size * audio_receiver->audio_header.channels * sizeof(opus_int16));
@ -95,5 +158,6 @@ CHIAKI_EXPORT void chiaki_audio_receiver_frame_packet(ChiakiAudioReceiver *audio
free(pcm);
beach:
chiaki_mutex_unlock(&audio_receiver->mutex);
}

27
lib/src/streamconnection.c
View file

@ -705,35 +705,10 @@ static void stream_connection_takion_av(ChiakiStreamConnection *stream_connectio
{
chiaki_gkcrypt_decrypt(stream_connection->gkcrypt_remote, packet->key_pos + CHIAKI_GKCRYPT_BLOCK_SIZE, packet->data, packet->data_size);
/*CHIAKI_LOGD(stream_connection->log, "AV: index: %u,%u; b@0x1a: %d; is_video: %d; 0xa: %u; 0xc: %u; 0xe: %u; codec: %u; 0x18: %u; adaptive_stream: %u, 0x2c: %u",
header->packet_index, header->frame_index, header->byte_at_0x1a, header->is_video ? 1 : 0, header->word_at_0xa, header->units_in_frame_total, header->units_in_frame_fec, header->codec,
header->word_at_0x18, header->adaptive_stream_index, header->byte_at_0x2c);
chiaki_log_hexdump(stream_connection->log, CHIAKI_LOG_DEBUG, buf, buf_size);*/
if(packet->is_video)
{
chiaki_video_receiver_av_packet(stream_connection->session->video_receiver, packet);
}
else
{
if(packet->codec == 5/*buf[0] == 0xf4 && buf_size >= 0x50*/)
{
//CHIAKI_LOGD(stream_connection->log, "audio!");
chiaki_audio_receiver_frame_packet(stream_connection->session->audio_receiver, packet->data, 0x50); // TODO: why 0x50? this is dangerous!!!
}
else
{
//CHIAKI_LOGD(stream_connection->log, "NON-audio");
}
}
/*else if(base_type == 2 && buf[0] != 0xf4)
{
CHIAKI_LOGD(stream_connection->log, "av frame 2, which is not audio");
}*/
//CHIAKI_LOGD(stream_connection->log, "StreamConnection AV %lu", buf_size);
//chiaki_log_hexdump(stream_connection->log, CHIAKI_LOG_DEBUG, buf, buf_size);
chiaki_audio_receiver_av_packet(stream_connection->session->audio_receiver, packet);
}