Implement Audio FEC

lib/include/chiaki/audioreceiver.h

@@ -21,6 +21,7 @@
 #include "common.h"
 #include "log.h"
 #include "audio.h"
+#include "takion.h"
 #include "thread.h"
 
 #ifdef __cplusplus
@@ -35,12 +36,14 @@ typedef struct chiaki_audio_receiver_t
 	ChiakiMutex mutex;
 	struct OpusDecoder *opus_decoder;
 	ChiakiAudioHeader audio_header;
+	ChiakiSeqNum16 frame_index_prev;
+	bool frame_index_startup; // whether frame_index_prev has definitely not wrapped yet
 } ChiakiAudioReceiver;
 
 CHIAKI_EXPORT ChiakiErrorCode chiaki_audio_receiver_init(ChiakiAudioReceiver *audio_receiver, struct chiaki_session_t *session);
 CHIAKI_EXPORT void chiaki_audio_receiver_fini(ChiakiAudioReceiver *audio_receiver);
 CHIAKI_EXPORT void chiaki_audio_receiver_stream_info(ChiakiAudioReceiver *audio_receiver, ChiakiAudioHeader *audio_header);
-CHIAKI_EXPORT void chiaki_audio_receiver_frame_packet(ChiakiAudioReceiver *audio_receiver, uint8_t *buf, size_t buf_size);
+CHIAKI_EXPORT void chiaki_audio_receiver_av_packet(ChiakiAudioReceiver *audio_receiver, ChiakiTakionAVPacket *packet);
 
 static inline ChiakiAudioReceiver *chiaki_audio_receiver_new(struct chiaki_session_t *session)
 {

lib/include/chiaki/takion.h

@@ -63,6 +63,10 @@ typedef struct chiaki_takion_av_packet_t
 	size_t data_size;
 } ChiakiTakionAVPacket;
 
+static inline uint8_t chiaki_takion_av_packet_audio_unit_size(ChiakiTakionAVPacket *packet)				{ return packet->units_in_frame_fec >> 8; }
+static inline uint8_t chiaki_takion_av_packet_audio_source_units_count(ChiakiTakionAVPacket *packet)	{ return packet->units_in_frame_fec & 0xf; }
+static inline uint8_t chiaki_takion_av_packet_audio_fec_units_count(ChiakiTakionAVPacket *packet)		{ return (packet->units_in_frame_fec >> 4) & 0xf; }
+
 
 typedef struct chiaki_takion_congestion_packet_t
 {

lib/src/audioreceiver.c

@@ -22,6 +22,7 @@
 
 #include <string.h>
 
+static void chiaki_audio_receiver_frame(ChiakiAudioReceiver *audio_receiver, ChiakiSeqNum16 frame_index, uint8_t *buf, size_t buf_size);
 
 CHIAKI_EXPORT ChiakiErrorCode chiaki_audio_receiver_init(ChiakiAudioReceiver *audio_receiver, ChiakiSession *session)
 {
@@ -30,6 +31,9 @@ CHIAKI_EXPORT ChiakiErrorCode chiaki_audio_receiver_init(ChiakiAudioReceiver *au
 	audio_receiver->opus_decoder = NULL;
 	memset(&audio_receiver->audio_header, 0, sizeof(audio_receiver->audio_header));
 
+	audio_receiver->frame_index_prev = 0;
+	audio_receiver->frame_index_startup = true;
+
 	ChiakiErrorCode err = chiaki_mutex_init(&audio_receiver->mutex, false);
 	if(err != CHIAKI_ERR_SUCCESS)
 		return err;
@@ -70,18 +74,77 @@ CHIAKI_EXPORT void chiaki_audio_receiver_stream_info(ChiakiAudioReceiver *audio_
 	chiaki_mutex_unlock(&audio_receiver->mutex);
 }
 
+CHIAKI_EXPORT void chiaki_audio_receiver_av_packet(ChiakiAudioReceiver *audio_receiver, ChiakiTakionAVPacket *packet)
+{
+	if(packet->codec != 5)
+	{
+		CHIAKI_LOGE(audio_receiver->log, "Received Audio Packet with unknown Codec");
+		return;
+	}
 
-CHIAKI_EXPORT void chiaki_audio_receiver_frame_packet(ChiakiAudioReceiver *audio_receiver, uint8_t *buf, size_t buf_size)
+	// this is mostly observation-based, so may not necessarily cover everything yet.
+
+	uint8_t source_units_count = chiaki_takion_av_packet_audio_source_units_count(packet);
+	uint8_t fec_units_count = chiaki_takion_av_packet_audio_fec_units_count(packet);
+	uint8_t unit_size = chiaki_takion_av_packet_audio_unit_size(packet);
+
+	if(!packet->data_size)
+	{
+		CHIAKI_LOGE(audio_receiver->log, "Audio AV Packet is empty");
+		return;
+	}
+
+	if((uint16_t)fec_units_count + (uint16_t)source_units_count != packet->units_in_frame_total)
+	{
+		CHIAKI_LOGE(audio_receiver->log, "Source Units + FEC Units != Total Units in Audio AV Packet");
+		return;
+	}
+
+	if(packet->data_size != (size_t)unit_size * (size_t)packet->units_in_frame_total)
+	{
+		CHIAKI_LOGE(audio_receiver->log, "Audio AV Packet size mismatch");
+		return;
+	}
+
+	if(packet->frame_index > (1 << 15))
+		audio_receiver->frame_index_startup = false;
+
+	for(size_t i=0; i<source_units_count+fec_units_count; i++)
+	{
+		ChiakiSeqNum16 frame_index;
+		if(i < source_units_count)
+			frame_index = packet->frame_index + i;
+		else
+		{
+			// fec
+			size_t fec_index = i - source_units_count;
+
+			// first packets will contain the same frame multiple times, ignore those
+			if(audio_receiver->frame_index_startup && packet->frame_index + fec_index < fec_units_count + 1)
+				continue;
+
+			frame_index = packet->frame_index - fec_units_count + fec_index;
+		}
+
+		chiaki_audio_receiver_frame(audio_receiver->session->audio_receiver, frame_index, packet->data + unit_size * i, unit_size);
+	}
+}
+
+static void chiaki_audio_receiver_frame(ChiakiAudioReceiver *audio_receiver, ChiakiSeqNum16 frame_index, uint8_t *buf, size_t buf_size)
 {
 	chiaki_mutex_lock(&audio_receiver->mutex);
 
 	if(!audio_receiver->opus_decoder)
 	{
 		CHIAKI_LOGE(audio_receiver->log, "Received audio frame, but opus decoder is not initialized");
-		chiaki_mutex_unlock(&audio_receiver->mutex);
+		goto beach;
-		return;
 	}
 
+	if(!chiaki_seq_num_16_gt(frame_index, audio_receiver->frame_index_prev))
+		goto beach;
+
+	audio_receiver->frame_index_prev = frame_index;
+
 	// TODO: don't malloc
 	opus_int16 *pcm = malloc(audio_receiver->audio_header.frame_size * audio_receiver->audio_header.channels * sizeof(opus_int16));
 
@@ -95,5 +158,6 @@ CHIAKI_EXPORT void chiaki_audio_receiver_frame_packet(ChiakiAudioReceiver *audio
 
 	free(pcm);
 
+beach:
 	chiaki_mutex_unlock(&audio_receiver->mutex);
 }

lib/src/streamconnection.c

@@ -705,35 +705,10 @@ static void stream_connection_takion_av(ChiakiStreamConnection *stream_connectio
 {
 	chiaki_gkcrypt_decrypt(stream_connection->gkcrypt_remote, packet->key_pos + CHIAKI_GKCRYPT_BLOCK_SIZE, packet->data, packet->data_size);
 
-	/*CHIAKI_LOGD(stream_connection->log, "AV: index: %u,%u; b@0x1a: %d; is_video: %d; 0xa: %u; 0xc: %u; 0xe: %u; codec: %u; 0x18: %u; adaptive_stream: %u, 0x2c: %u",
-				header->packet_index, header->frame_index, header->byte_at_0x1a, header->is_video ? 1 : 0, header->word_at_0xa, header->units_in_frame_total, header->units_in_frame_fec, header->codec,
-				header->word_at_0x18, header->adaptive_stream_index, header->byte_at_0x2c);
-	chiaki_log_hexdump(stream_connection->log, CHIAKI_LOG_DEBUG, buf, buf_size);*/
-
 	if(packet->is_video)
-	{
 		chiaki_video_receiver_av_packet(stream_connection->session->video_receiver, packet);
-	}
 	else
-	{
+		chiaki_audio_receiver_av_packet(stream_connection->session->audio_receiver, packet);
-		if(packet->codec == 5/*buf[0] == 0xf4 && buf_size >= 0x50*/)
-		{
-			//CHIAKI_LOGD(stream_connection->log, "audio!");
-			chiaki_audio_receiver_frame_packet(stream_connection->session->audio_receiver, packet->data, 0x50); // TODO: why 0x50? this is dangerous!!!
-		}
-		else
-		{
-			//CHIAKI_LOGD(stream_connection->log, "NON-audio");
-		}
-	}
-
-	/*else if(base_type == 2 && buf[0] != 0xf4)
-	{
-		CHIAKI_LOGD(stream_connection->log, "av frame 2, which is not audio");
-	}*/
-
-	//CHIAKI_LOGD(stream_connection->log, "StreamConnection AV %lu", buf_size);
-	//chiaki_log_hexdump(stream_connection->log, CHIAKI_LOG_DEBUG, buf, buf_size);
 }