github/chiaki
1
0
Fork 0
mirror of https://git.sr.ht/~thestr4ng3r/chiaki synced 2025-08-19 13:09:39 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add Early Video Frame Flush

Browse source
This commit is contained in:
Florian Märkl 2019-07-29 22:10:28 +02:00
commit ac87d622c8
No known key found for this signature in database
GPG key ID: 125BC8A5A6A1E857
7 changed files with 55 additions and 39 deletions
Download patch file Download diff file Expand all files Collapse all files

15
lib/include/chiaki/frameprocessor.h
View file

@ -22,6 +22,7 @@
#include "takion.h" #include "takion.h"
#include <stdint.h> #include <stdint.h>
#include <stdbool.h>
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
@ -36,10 +37,10 @@ typedef struct chiaki_frame_processor_t
uint8_t *frame_buf; uint8_t *frame_buf;
size_t frame_buf_size; size_t frame_buf_size;
size_t buf_size_per_unit; size_t buf_size_per_unit;
unsigned int units_regular_expected; unsigned int units_source_expected;
unsigned int units_additional_expected; unsigned int units_fec_expected;
unsigned int units_regular_received; unsigned int units_source_received;
unsigned int units_additional_received; unsigned int units_fec_received;
ChiakiFrameUnit *unit_slots; ChiakiFrameUnit *unit_slots;
size_t unit_slots_size; size_t unit_slots_size;
} ChiakiFrameProcessor; } ChiakiFrameProcessor;
@ -57,6 +58,12 @@ CHIAKI_EXPORT ChiakiErrorCode chiaki_frame_processor_alloc_frame(ChiakiFrameProc
CHIAKI_EXPORT ChiakiErrorCode chiaki_frame_processor_put_unit(ChiakiFrameProcessor *frame_processor, ChiakiTakionAVPacket *packet); CHIAKI_EXPORT ChiakiErrorCode chiaki_frame_processor_put_unit(ChiakiFrameProcessor *frame_processor, ChiakiTakionAVPacket *packet);
CHIAKI_EXPORT ChiakiFrameProcessorFlushResult chiaki_frame_processor_flush(ChiakiFrameProcessor *frame_processor, uint8_t **frame, size_t *frame_size); CHIAKI_EXPORT ChiakiFrameProcessorFlushResult chiaki_frame_processor_flush(ChiakiFrameProcessor *frame_processor, uint8_t **frame, size_t *frame_size);
static inline bool chiaki_frame_processor_flush_possible(ChiakiFrameProcessor *frame_processor)
{
return frame_processor->units_source_received //+ frame_processor->units_fec_received
>= frame_processor->units_source_expected;
}
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

4
lib/include/chiaki/takion.h
View file

@ -50,8 +50,8 @@ typedef struct chiaki_takion_av_packet_t
bool uses_nalu_info_structs; bool uses_nalu_info_structs;
bool is_video; bool is_video;
ChiakiSeqNum16 unit_index; ChiakiSeqNum16 unit_index;
uint16_t units_in_frame_total; // regular + units_in_frame_additional uint16_t units_in_frame_total; // source + units_in_frame_fec
uint16_t units_in_frame_additional; uint16_t units_in_frame_fec;
uint32_t codec; uint32_t codec;
uint16_t word_at_0x18; uint16_t word_at_0x18;
uint8_t adaptive_stream_index; uint8_t adaptive_stream_index;

50
lib/src/frameprocessor.c
View file

@ -36,8 +36,8 @@ CHIAKI_EXPORT void chiaki_frame_processor_init(ChiakiFrameProcessor *frame_proce
frame_processor->log = log; frame_processor->log = log;
frame_processor->frame_buf = NULL; frame_processor->frame_buf = NULL;
frame_processor->frame_buf_size = 0; frame_processor->frame_buf_size = 0;
frame_processor->units_regular_expected = 0; frame_processor->units_source_expected = 0;
frame_processor->units_additional_expected = 0; frame_processor->units_fec_expected = 0;
frame_processor->unit_slots = NULL; frame_processor->unit_slots = NULL;
frame_processor->unit_slots_size = 0; frame_processor->unit_slots_size = 0;
} }
@ -50,19 +50,19 @@ CHIAKI_EXPORT void chiaki_frame_processor_fini(ChiakiFrameProcessor *frame_proce
CHIAKI_EXPORT ChiakiErrorCode chiaki_frame_processor_alloc_frame(ChiakiFrameProcessor *frame_processor, ChiakiTakionAVPacket *packet) CHIAKI_EXPORT ChiakiErrorCode chiaki_frame_processor_alloc_frame(ChiakiFrameProcessor *frame_processor, ChiakiTakionAVPacket *packet)
{ {
if(packet->units_in_frame_total < packet->units_in_frame_additional) if(packet->units_in_frame_total < packet->units_in_frame_fec)
{ {
CHIAKI_LOGE(frame_processor->log, "Packet has units_in_frame_total < units_in_frame_additional"); CHIAKI_LOGE(frame_processor->log, "Packet has units_in_frame_total < units_in_frame_fec");
return CHIAKI_ERR_INVALID_DATA; return CHIAKI_ERR_INVALID_DATA;
} }
frame_processor->units_regular_expected = packet->units_in_frame_total - packet->units_in_frame_additional; frame_processor->units_source_expected = packet->units_in_frame_total - packet->units_in_frame_fec;
frame_processor->units_additional_expected = packet->units_in_frame_additional; frame_processor->units_fec_expected = packet->units_in_frame_fec;
if(frame_processor->units_additional_expected < 1) if(frame_processor->units_fec_expected < 1)
frame_processor->units_additional_expected = 1; frame_processor->units_fec_expected = 1;
frame_processor->buf_size_per_unit = packet->data_size; frame_processor->buf_size_per_unit = packet->data_size;
if(packet->is_video && packet->unit_index < frame_processor->units_regular_expected) if(packet->is_video && packet->unit_index < frame_processor->units_source_expected)
{ {
if(packet->data_size < 2) if(packet->data_size < 2)
{ {
@ -78,10 +78,10 @@ CHIAKI_EXPORT ChiakiErrorCode chiaki_frame_processor_alloc_frame(ChiakiFrameProc
return CHIAKI_ERR_BUF_TOO_SMALL; return CHIAKI_ERR_BUF_TOO_SMALL;
} }
frame_processor->units_regular_received = 0; frame_processor->units_source_received = 0;
frame_processor->units_additional_received = 0; frame_processor->units_fec_received = 0;
size_t unit_slots_size_required = frame_processor->units_regular_expected + frame_processor->units_additional_expected; size_t unit_slots_size_required = frame_processor->units_source_expected + frame_processor->units_fec_expected;
if(unit_slots_size_required > UNIT_SLOTS_MAX) if(unit_slots_size_required > UNIT_SLOTS_MAX)
{ {
CHIAKI_LOGE(frame_processor->log, "Packet suggests more than %u unit slots", UNIT_SLOTS_MAX); CHIAKI_LOGE(frame_processor->log, "Packet suggests more than %u unit slots", UNIT_SLOTS_MAX);
@ -161,10 +161,10 @@ CHIAKI_EXPORT ChiakiErrorCode chiaki_frame_processor_put_unit(ChiakiFrameProcess
packet->data, packet->data,
packet->data_size); packet->data_size);
if(packet->unit_index < frame_processor->units_regular_expected) if(packet->unit_index < frame_processor->units_source_expected)
frame_processor->units_regular_received++; frame_processor->units_source_received++;
else else
frame_processor->units_additional_received++; frame_processor->units_fec_received++;
return CHIAKI_ERR_SUCCESS; return CHIAKI_ERR_SUCCESS;
} }
@ -175,18 +175,18 @@ CHIAKI_EXPORT ChiakiErrorCode chiaki_frame_processor_put_unit(ChiakiFrameProcess
static ChiakiErrorCode chiaki_frame_processor_fec(ChiakiFrameProcessor *frame_processor) static ChiakiErrorCode chiaki_frame_processor_fec(ChiakiFrameProcessor *frame_processor)
{ {
CHIAKI_LOGI(frame_processor->log, "Frame Processor received %u+%u / %u+%u units, attempting FEC", CHIAKI_LOGI(frame_processor->log, "Frame Processor received %u+%u / %u+%u units, attempting FEC",
frame_processor->units_regular_received, frame_processor->units_additional_received, frame_processor->units_source_received, frame_processor->units_fec_received,
frame_processor->units_regular_expected, frame_processor->units_additional_expected); frame_processor->units_source_expected, frame_processor->units_fec_expected);
size_t erasures_count = (frame_processor->units_regular_expected + frame_processor->units_additional_expected) size_t erasures_count = (frame_processor->units_source_expected + frame_processor->units_fec_expected)
- (frame_processor->units_regular_received + frame_processor->units_additional_received); - (frame_processor->units_source_received + frame_processor->units_fec_received);
unsigned int *erasures = calloc(erasures_count, sizeof(unsigned int)); unsigned int *erasures = calloc(erasures_count, sizeof(unsigned int));
if(!erasures) if(!erasures)
return CHIAKI_ERR_MEMORY; return CHIAKI_ERR_MEMORY;
size_t erasure_index = 0; size_t erasure_index = 0;
for(size_t i=0; i<frame_processor->units_regular_expected + frame_processor->units_additional_expected; i++) for(size_t i=0; i<frame_processor->units_source_expected + frame_processor->units_fec_expected; i++)
{ {
ChiakiFrameUnit *slot = frame_processor->unit_slots + i; ChiakiFrameUnit *slot = frame_processor->unit_slots + i;
if(!slot->data_size) if(!slot->data_size)
@ -204,7 +204,7 @@ static ChiakiErrorCode chiaki_frame_processor_fec(ChiakiFrameProcessor *frame_pr
assert(erasure_index == erasures_count); assert(erasure_index == erasures_count);
ChiakiErrorCode err = chiaki_fec_decode(frame_processor->frame_buf, frame_processor->buf_size_per_unit, ChiakiErrorCode err = chiaki_fec_decode(frame_processor->frame_buf, frame_processor->buf_size_per_unit,
frame_processor->units_regular_expected, frame_processor->units_additional_received, frame_processor->units_source_expected, frame_processor->units_fec_received,
erasures, erasures_count); erasures, erasures_count);
if(err != CHIAKI_ERR_SUCCESS) if(err != CHIAKI_ERR_SUCCESS)
@ -218,7 +218,7 @@ static ChiakiErrorCode chiaki_frame_processor_fec(ChiakiFrameProcessor *frame_pr
CHIAKI_LOGI(frame_processor->log, "FEC successful"); CHIAKI_LOGI(frame_processor->log, "FEC successful");
// restore unit sizes // restore unit sizes
for(size_t i=0; i<frame_processor->units_regular_expected; i++) for(size_t i=0; i<frame_processor->units_source_expected; i++)
{ {
ChiakiFrameUnit *slot = frame_processor->unit_slots + i; ChiakiFrameUnit *slot = frame_processor->unit_slots + i;
uint8_t *buf_ptr = frame_processor->frame_buf + frame_processor->buf_size_per_unit * i; uint8_t *buf_ptr = frame_processor->frame_buf + frame_processor->buf_size_per_unit * i;
@ -240,10 +240,10 @@ static ChiakiErrorCode chiaki_frame_processor_fec(ChiakiFrameProcessor *frame_pr
CHIAKI_EXPORT ChiakiFrameProcessorFlushResult chiaki_frame_processor_flush(ChiakiFrameProcessor *frame_processor, uint8_t **frame, size_t *frame_size) CHIAKI_EXPORT ChiakiFrameProcessorFlushResult chiaki_frame_processor_flush(ChiakiFrameProcessor *frame_processor, uint8_t **frame, size_t *frame_size)
{ {
if(frame_processor->units_regular_expected == 0) if(frame_processor->units_source_expected == 0)
return CHIAKI_FRAME_PROCESSOR_FLUSH_RESULT_FAILED; return CHIAKI_FRAME_PROCESSOR_FLUSH_RESULT_FAILED;
if(frame_processor->units_regular_received < frame_processor->units_regular_expected) if(frame_processor->units_source_received < frame_processor->units_source_expected)
{ {
ChiakiErrorCode err = chiaki_frame_processor_fec(frame_processor); ChiakiErrorCode err = chiaki_frame_processor_fec(frame_processor);
if(err != CHIAKI_ERR_SUCCESS) if(err != CHIAKI_ERR_SUCCESS)
@ -255,7 +255,7 @@ CHIAKI_EXPORT ChiakiFrameProcessorFlushResult chiaki_frame_processor_flush(Chiak
return CHIAKI_FRAME_PROCESSOR_FLUSH_RESULT_FAILED; return CHIAKI_FRAME_PROCESSOR_FLUSH_RESULT_FAILED;
size_t buf_size = 0; size_t buf_size = 0;
for(size_t i=0; i<frame_processor->units_regular_expected; i++) for(size_t i=0; i<frame_processor->units_source_expected; i++)
{ {
ChiakiFrameUnit *unit = frame_processor->unit_slots + i; ChiakiFrameUnit *unit = frame_processor->unit_slots + i;
if(unit->data_size < 2) if(unit->data_size < 2)

2
lib/src/streamconnection.c
View file

@ -706,7 +706,7 @@ static void stream_connection_takion_av(ChiakiStreamConnection *stream_connectio
chiaki_gkcrypt_decrypt(stream_connection->gkcrypt_remote, packet->key_pos + CHIAKI_GKCRYPT_BLOCK_SIZE, packet->data, packet->data_size); chiaki_gkcrypt_decrypt(stream_connection->gkcrypt_remote, packet->key_pos + CHIAKI_GKCRYPT_BLOCK_SIZE, packet->data, packet->data_size);
/*CHIAKI_LOGD(stream_connection->log, "AV: index: %u,%u; b@0x1a: %d; is_video: %d; 0xa: %u; 0xc: %u; 0xe: %u; codec: %u; 0x18: %u; adaptive_stream: %u, 0x2c: %u", /*CHIAKI_LOGD(stream_connection->log, "AV: index: %u,%u; b@0x1a: %d; is_video: %d; 0xa: %u; 0xc: %u; 0xe: %u; codec: %u; 0x18: %u; adaptive_stream: %u, 0x2c: %u",
header->packet_index, header->frame_index, header->byte_at_0x1a, header->is_video ? 1 : 0, header->word_at_0xa, header->units_in_frame_total, header->units_in_frame_additional, header->codec, header->packet_index, header->frame_index, header->byte_at_0x1a, header->is_video ? 1 : 0, header->word_at_0xa, header->units_in_frame_total, header->units_in_frame_fec, header->codec,
header->word_at_0x18, header->adaptive_stream_index, header->byte_at_0x2c); header->word_at_0x18, header->adaptive_stream_index, header->byte_at_0x2c);
chiaki_log_hexdump(stream_connection->log, CHIAKI_LOG_DEBUG, buf, buf_size);*/ chiaki_log_hexdump(stream_connection->log, CHIAKI_LOG_DEBUG, buf, buf_size);*/

4
lib/src/takion.c
View file

@ -1146,13 +1146,13 @@ CHIAKI_EXPORT ChiakiErrorCode chiaki_takion_av_packet_parse(ChiakiTakionAVPacket
{ {
packet->unit_index = (uint16_t)((dword_2 >> 0x15) & 0x7ff); packet->unit_index = (uint16_t)((dword_2 >> 0x15) & 0x7ff);
packet->units_in_frame_total = (uint16_t)(((dword_2 >> 0xa) & 0x7ff) + 1); packet->units_in_frame_total = (uint16_t)(((dword_2 >> 0xa) & 0x7ff) + 1);
packet->units_in_frame_additional = (uint16_t)(dword_2 & 0x3ff); packet->units_in_frame_fec = (uint16_t)(dword_2 & 0x3ff);
} }
else else
{ {
packet->unit_index = (uint16_t)((dword_2 >> 0x18) & 0xff); packet->unit_index = (uint16_t)((dword_2 >> 0x18) & 0xff);
packet->units_in_frame_total = (uint16_t)(((dword_2 >> 0x10) & 0xff) + 1); packet->units_in_frame_total = (uint16_t)(((dword_2 >> 0x10) & 0xff) + 1);
packet->units_in_frame_additional = (uint16_t)(dword_2 & 0xffff); packet->units_in_frame_fec = (uint16_t)(dword_2 & 0xffff);
} }
packet->codec = av[8]; packet->codec = av[8];

17
lib/src/videoreceiver.c
View file

@ -20,7 +20,7 @@
#include <string.h> #include <string.h>
static void chiaki_video_receiver_flush_frame(ChiakiVideoReceiver *video_receiver); static ChiakiErrorCode chiaki_video_receiver_flush_frame(ChiakiVideoReceiver *video_receiver);
CHIAKI_EXPORT void chiaki_video_receiver_init(ChiakiVideoReceiver *video_receiver, struct chiaki_session_t *session) CHIAKI_EXPORT void chiaki_video_receiver_init(ChiakiVideoReceiver *video_receiver, struct chiaki_session_t *session)
{ {
@ -111,10 +111,18 @@ CHIAKI_EXPORT void chiaki_video_receiver_av_packet(ChiakiVideoReceiver *video_re
chiaki_frame_processor_alloc_frame(&video_receiver->frame_processor, packet); chiaki_frame_processor_alloc_frame(&video_receiver->frame_processor, packet);
} }
chiaki_frame_processor_put_unit(&video_receiver->frame_processor, packet); // if we are currently building up a frame
if(video_receiver->frame_index_cur != video_receiver->frame_index_prev)
{
chiaki_frame_processor_put_unit(&video_receiver->frame_processor, packet);
// if we already have enough for the whole frame, flush it already
if(chiaki_frame_processor_flush_possible(&video_receiver->frame_processor))
chiaki_video_receiver_flush_frame(video_receiver);
}
} }
static void chiaki_video_receiver_flush_frame(ChiakiVideoReceiver *video_receiver) static ChiakiErrorCode chiaki_video_receiver_flush_frame(ChiakiVideoReceiver *video_receiver)
{ {
uint8_t *frame; uint8_t *frame;
size_t frame_size; size_t frame_size;
@ -124,11 +132,12 @@ static void chiaki_video_receiver_flush_frame(ChiakiVideoReceiver *video_receive
{ {
// TODO: fake frame? // TODO: fake frame?
CHIAKI_LOGW(video_receiver->log, "Failed to complete frame %d", (int)video_receiver->frame_index_cur); CHIAKI_LOGW(video_receiver->log, "Failed to complete frame %d", (int)video_receiver->frame_index_cur);
return; return CHIAKI_ERR_UNKNOWN;
} }
if(video_receiver->session->video_sample_cb) if(video_receiver->session->video_sample_cb)
video_receiver->session->video_sample_cb(frame, frame_size, video_receiver->session->video_sample_cb_user); video_receiver->session->video_sample_cb(frame, frame_size, video_receiver->session->video_sample_cb_user);
free(frame); free(frame);
video_receiver->frame_index_prev = video_receiver->frame_index_cur; video_receiver->frame_index_prev = video_receiver->frame_index_cur;
return CHIAKI_ERR_SUCCESS;
} }

2
test/takion.c
View file

@ -48,7 +48,7 @@ static MunitResult test_av_packet_parse(const MunitParameter params[], void *use
// TODO: uses_nalu_info_structs // TODO: uses_nalu_info_structs
munit_assert_uint16(av_packet.unit_index, ==, 6); munit_assert_uint16(av_packet.unit_index, ==, 6);
munit_assert_uint16(av_packet.units_in_frame_total, ==, 8); munit_assert_uint16(av_packet.units_in_frame_total, ==, 8);
munit_assert_uint16(av_packet.units_in_frame_additional, ==, 1); munit_assert_uint16(av_packet.units_in_frame_fec, ==, 1);
munit_assert_uint32(av_packet.codec, ==, 3); munit_assert_uint32(av_packet.codec, ==, 3);
// munit_assert_uint16(av_packet.word_at_0x18, ==, 871); // munit_assert_uint16(av_packet.word_at_0x18, ==, 871);
munit_assert_uint8(av_packet.adaptive_stream_index, ==, 0); munit_assert_uint8(av_packet.adaptive_stream_index, ==, 0);