diff --git a/lib/include/chiaki/audioreceiver.h b/lib/include/chiaki/audioreceiver.h index c325b7f..aac96ef 100644 --- a/lib/include/chiaki/audioreceiver.h +++ b/lib/include/chiaki/audioreceiver.h @@ -21,6 +21,7 @@ #include "common.h" #include "log.h" #include "audio.h" +#include "takion.h" #include "thread.h" #ifdef __cplusplus @@ -35,12 +36,14 @@ typedef struct chiaki_audio_receiver_t ChiakiMutex mutex; struct OpusDecoder *opus_decoder; ChiakiAudioHeader audio_header; + ChiakiSeqNum16 frame_index_prev; + bool frame_index_startup; // whether frame_index_prev has definitely not wrapped yet } ChiakiAudioReceiver; CHIAKI_EXPORT ChiakiErrorCode chiaki_audio_receiver_init(ChiakiAudioReceiver *audio_receiver, struct chiaki_session_t *session); CHIAKI_EXPORT void chiaki_audio_receiver_fini(ChiakiAudioReceiver *audio_receiver); CHIAKI_EXPORT void chiaki_audio_receiver_stream_info(ChiakiAudioReceiver *audio_receiver, ChiakiAudioHeader *audio_header); -CHIAKI_EXPORT void chiaki_audio_receiver_frame_packet(ChiakiAudioReceiver *audio_receiver, uint8_t *buf, size_t buf_size); +CHIAKI_EXPORT void chiaki_audio_receiver_av_packet(ChiakiAudioReceiver *audio_receiver, ChiakiTakionAVPacket *packet); static inline ChiakiAudioReceiver *chiaki_audio_receiver_new(struct chiaki_session_t *session) { diff --git a/lib/include/chiaki/takion.h b/lib/include/chiaki/takion.h index a12b440..ab02800 100644 --- a/lib/include/chiaki/takion.h +++ b/lib/include/chiaki/takion.h @@ -63,6 +63,10 @@ typedef struct chiaki_takion_av_packet_t size_t data_size; } ChiakiTakionAVPacket; +static inline uint8_t chiaki_takion_av_packet_audio_unit_size(ChiakiTakionAVPacket *packet) { return packet->units_in_frame_fec >> 8; } +static inline uint8_t chiaki_takion_av_packet_audio_source_units_count(ChiakiTakionAVPacket *packet) { return packet->units_in_frame_fec & 0xf; } +static inline uint8_t chiaki_takion_av_packet_audio_fec_units_count(ChiakiTakionAVPacket *packet) { return (packet->units_in_frame_fec >> 4) & 0xf; } + typedef struct chiaki_takion_congestion_packet_t { diff --git a/lib/src/audioreceiver.c b/lib/src/audioreceiver.c index d250388..7e58a6e 100644 --- a/lib/src/audioreceiver.c +++ b/lib/src/audioreceiver.c @@ -22,6 +22,7 @@ #include +static void chiaki_audio_receiver_frame(ChiakiAudioReceiver *audio_receiver, ChiakiSeqNum16 frame_index, uint8_t *buf, size_t buf_size); CHIAKI_EXPORT ChiakiErrorCode chiaki_audio_receiver_init(ChiakiAudioReceiver *audio_receiver, ChiakiSession *session) { @@ -30,6 +31,9 @@ CHIAKI_EXPORT ChiakiErrorCode chiaki_audio_receiver_init(ChiakiAudioReceiver *au audio_receiver->opus_decoder = NULL; memset(&audio_receiver->audio_header, 0, sizeof(audio_receiver->audio_header)); + audio_receiver->frame_index_prev = 0; + audio_receiver->frame_index_startup = true; + ChiakiErrorCode err = chiaki_mutex_init(&audio_receiver->mutex, false); if(err != CHIAKI_ERR_SUCCESS) return err; @@ -70,18 +74,77 @@ CHIAKI_EXPORT void chiaki_audio_receiver_stream_info(ChiakiAudioReceiver *audio_ chiaki_mutex_unlock(&audio_receiver->mutex); } +CHIAKI_EXPORT void chiaki_audio_receiver_av_packet(ChiakiAudioReceiver *audio_receiver, ChiakiTakionAVPacket *packet) +{ + if(packet->codec != 5) + { + CHIAKI_LOGE(audio_receiver->log, "Received Audio Packet with unknown Codec"); + return; + } -CHIAKI_EXPORT void chiaki_audio_receiver_frame_packet(ChiakiAudioReceiver *audio_receiver, uint8_t *buf, size_t buf_size) + // this is mostly observation-based, so may not necessarily cover everything yet. + + uint8_t source_units_count = chiaki_takion_av_packet_audio_source_units_count(packet); + uint8_t fec_units_count = chiaki_takion_av_packet_audio_fec_units_count(packet); + uint8_t unit_size = chiaki_takion_av_packet_audio_unit_size(packet); + + if(!packet->data_size) + { + CHIAKI_LOGE(audio_receiver->log, "Audio AV Packet is empty"); + return; + } + + if((uint16_t)fec_units_count + (uint16_t)source_units_count != packet->units_in_frame_total) + { + CHIAKI_LOGE(audio_receiver->log, "Source Units + FEC Units != Total Units in Audio AV Packet"); + return; + } + + if(packet->data_size != (size_t)unit_size * (size_t)packet->units_in_frame_total) + { + CHIAKI_LOGE(audio_receiver->log, "Audio AV Packet size mismatch"); + return; + } + + if(packet->frame_index > (1 << 15)) + audio_receiver->frame_index_startup = false; + + for(size_t i=0; iframe_index + i; + else + { + // fec + size_t fec_index = i - source_units_count; + + // first packets will contain the same frame multiple times, ignore those + if(audio_receiver->frame_index_startup && packet->frame_index + fec_index < fec_units_count + 1) + continue; + + frame_index = packet->frame_index - fec_units_count + fec_index; + } + + chiaki_audio_receiver_frame(audio_receiver->session->audio_receiver, frame_index, packet->data + unit_size * i, unit_size); + } +} + +static void chiaki_audio_receiver_frame(ChiakiAudioReceiver *audio_receiver, ChiakiSeqNum16 frame_index, uint8_t *buf, size_t buf_size) { chiaki_mutex_lock(&audio_receiver->mutex); if(!audio_receiver->opus_decoder) { CHIAKI_LOGE(audio_receiver->log, "Received audio frame, but opus decoder is not initialized"); - chiaki_mutex_unlock(&audio_receiver->mutex); - return; + goto beach; } + if(!chiaki_seq_num_16_gt(frame_index, audio_receiver->frame_index_prev)) + goto beach; + + audio_receiver->frame_index_prev = frame_index; + // TODO: don't malloc opus_int16 *pcm = malloc(audio_receiver->audio_header.frame_size * audio_receiver->audio_header.channels * sizeof(opus_int16)); @@ -95,5 +158,6 @@ CHIAKI_EXPORT void chiaki_audio_receiver_frame_packet(ChiakiAudioReceiver *audio free(pcm); +beach: chiaki_mutex_unlock(&audio_receiver->mutex); } \ No newline at end of file diff --git a/lib/src/streamconnection.c b/lib/src/streamconnection.c index be6cb00..09c8582 100644 --- a/lib/src/streamconnection.c +++ b/lib/src/streamconnection.c @@ -705,35 +705,10 @@ static void stream_connection_takion_av(ChiakiStreamConnection *stream_connectio { chiaki_gkcrypt_decrypt(stream_connection->gkcrypt_remote, packet->key_pos + CHIAKI_GKCRYPT_BLOCK_SIZE, packet->data, packet->data_size); - /*CHIAKI_LOGD(stream_connection->log, "AV: index: %u,%u; b@0x1a: %d; is_video: %d; 0xa: %u; 0xc: %u; 0xe: %u; codec: %u; 0x18: %u; adaptive_stream: %u, 0x2c: %u", - header->packet_index, header->frame_index, header->byte_at_0x1a, header->is_video ? 1 : 0, header->word_at_0xa, header->units_in_frame_total, header->units_in_frame_fec, header->codec, - header->word_at_0x18, header->adaptive_stream_index, header->byte_at_0x2c); - chiaki_log_hexdump(stream_connection->log, CHIAKI_LOG_DEBUG, buf, buf_size);*/ - if(packet->is_video) - { chiaki_video_receiver_av_packet(stream_connection->session->video_receiver, packet); - } else - { - if(packet->codec == 5/*buf[0] == 0xf4 && buf_size >= 0x50*/) - { - //CHIAKI_LOGD(stream_connection->log, "audio!"); - chiaki_audio_receiver_frame_packet(stream_connection->session->audio_receiver, packet->data, 0x50); // TODO: why 0x50? this is dangerous!!! - } - else - { - //CHIAKI_LOGD(stream_connection->log, "NON-audio"); - } - } - - /*else if(base_type == 2 && buf[0] != 0xf4) - { - CHIAKI_LOGD(stream_connection->log, "av frame 2, which is not audio"); - }*/ - - //CHIAKI_LOGD(stream_connection->log, "StreamConnection AV %lu", buf_size); - //chiaki_log_hexdump(stream_connection->log, CHIAKI_LOG_DEBUG, buf, buf_size); + chiaki_audio_receiver_av_packet(stream_connection->session->audio_receiver, packet); }