From 8876b591bc53607e111e5ee47709b85ff4c47e5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20M=C3=A4rkl?= Date: Mon, 10 Jun 2019 16:05:38 +0200 Subject: [PATCH] Implement Frame Processor --- gui/src/main.cpp | 20 ++- lib/CMakeLists.txt | 6 +- lib/include/chiaki/common.h | 1 + lib/include/chiaki/frameprocessor.h | 64 +++++++++ lib/include/chiaki/takion.h | 15 ++- lib/include/chiaki/videoreceiver.h | 8 +- lib/src/frameprocessor.c | 198 ++++++++++++++++++++++++++++ lib/src/nagare.c | 16 +-- lib/src/takion.c | 90 +++++++------ lib/src/videoreceiver.c | 82 ++++++++++-- 10 files changed, 432 insertions(+), 68 deletions(-) create mode 100644 lib/include/chiaki/frameprocessor.h create mode 100644 lib/src/frameprocessor.c diff --git a/gui/src/main.cpp b/gui/src/main.cpp index d47f18b..2b536a3 100644 --- a/gui/src/main.cpp +++ b/gui/src/main.cpp @@ -15,6 +15,9 @@ QAudioOutput *audio_out; QIODevice *audio_io; +QFile *video_out_file; +size_t file_size = 0; + void audio_frame_cb(int16_t *buf, size_t samples_count, void *user) { @@ -23,6 +26,11 @@ void audio_frame_cb(int16_t *buf, size_t samples_count, void *user) void video_sample_cb(uint8_t *buf, size_t buf_size, void *user) { + if(!video_out_file) + return; + printf("writing %#zx to file, start: %#zx\n", buf_size, file_size); + file_size += buf_size; + video_out_file->write((const char *)buf, buf_size); //StreamRelayIODevice *io_device = reinterpret_cast(user); //io_device->PushSample(buf, buf_size); } @@ -89,18 +97,28 @@ int main(int argc, char *argv[]) audio_io = audio_out->start(); + video_out_file = nullptr; + //video_out_file->open(QFile::ReadWrite); + ChiakiSession session; chiaki_session_init(&session, &connect_info); chiaki_session_set_audio_frame_cb(&session, audio_frame_cb, NULL); chiaki_session_set_video_sample_cb(&session, video_sample_cb, window.GetIODevice()); chiaki_session_start(&session); + app.setQuitOnLastWindowClosed(true); + int ret = app.exec(); + //video_out_file->close(); + + printf("CLOSED!!! filesize: %zu\n", file_size); + chiaki_session_join(&session); chiaki_session_fini(&session); delete audio_out; + return ret; -} \ No newline at end of file +} diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index 19bae14..679576a 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -19,7 +19,8 @@ set(HEADER_FILES include/chiaki/audio.h include/chiaki/audioreceiver.h include/chiaki/video.h - include/chiaki/videoreceiver.h) + include/chiaki/videoreceiver.h + include/chiaki/frameprocessor.h) set(SOURCE_FILES src/common.c @@ -42,7 +43,8 @@ set(SOURCE_FILES src/gkcrypt.c src/audio.c src/audioreceiver.c - src/videoreceiver.c) + src/videoreceiver.c + src/frameprocessor.c) add_subdirectory(protobuf) include_directories("${NANOPB_SOURCE_DIR}") diff --git a/lib/include/chiaki/common.h b/lib/include/chiaki/common.h index 20cdd1a..aa58952 100644 --- a/lib/include/chiaki/common.h +++ b/lib/include/chiaki/common.h @@ -33,6 +33,7 @@ typedef enum CHIAKI_ERR_PARSE_ADDR, CHIAKI_ERR_THREAD, CHIAKI_ERR_MEMORY, + CHIAKI_ERR_OVERFLOW, CHIAKI_ERR_NETWORK, CHIAKI_ERR_INVALID_DATA, CHIAKI_ERR_BUF_TOO_SMALL, diff --git a/lib/include/chiaki/frameprocessor.h b/lib/include/chiaki/frameprocessor.h new file mode 100644 index 0000000..2945228 --- /dev/null +++ b/lib/include/chiaki/frameprocessor.h @@ -0,0 +1,64 @@ +/* + * This file is part of Chiaki. + * + * Chiaki is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Chiaki is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Chiaki. If not, see . + */ + +#ifndef CHIAKI_FRAMEPROCESSOR_H +#define CHIAKI_FRAMEPROCESSOR_H + +#include "common.h" +#include "takion.h" + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +struct chiaki_frame_unit_t; +typedef struct chiaki_frame_unit_t ChiakiFrameUnit; + +typedef struct chiaki_frame_processor_t +{ + ChiakiLog *log; + uint8_t *frame_buf; + size_t frame_buf_size; + size_t buf_size_per_unit; + unsigned int units_regular_expected; + unsigned int units_additional_expected; + unsigned int units_regular_received; + unsigned int units_additional_received; + ChiakiFrameUnit *unit_slots; + size_t unit_slots_size; +} ChiakiFrameProcessor; + +typedef enum chiaki_frame_flush_result_t { + CHIAKI_FRAME_PROCESSOR_FLUSH_RESULT_SUCCESS = 0, + CHIAKI_FRAME_PROCESSOR_FLUSH_RESULT_FAILED = 1 + // TODO: FEC_SUCCESS, FEC_FAILED, ... +} ChiakiFrameProcessorFlushResult; + +CHIAKI_EXPORT void chiaki_frame_processor_init(ChiakiFrameProcessor *frame_processor, ChiakiLog *log); +CHIAKI_EXPORT void chiaki_frame_processor_fini(ChiakiFrameProcessor *frame_processor); + +CHIAKI_EXPORT ChiakiErrorCode chiaki_frame_processor_alloc_frame(ChiakiFrameProcessor *frame_processor, ChiakiTakionAVPacket *packet); +CHIAKI_EXPORT ChiakiErrorCode chiaki_frame_processor_put_unit(ChiakiFrameProcessor *frame_processor, ChiakiTakionAVPacket *packet); +CHIAKI_EXPORT ChiakiFrameProcessorFlushResult chiaki_frame_processor_flush(ChiakiFrameProcessor *frame_processor, uint8_t **frame, size_t *frame_size); + +#ifdef __cplusplus +} +#endif + +#endif // CHIAKI_FRAMEPROCESSOR_H diff --git a/lib/include/chiaki/takion.h b/lib/include/chiaki/takion.h index 85b9859..a0e2abc 100644 --- a/lib/include/chiaki/takion.h +++ b/lib/include/chiaki/takion.h @@ -34,7 +34,7 @@ extern "C" { struct chiaki_takion_av_packet_t; typedef void (*ChiakiTakionDataCallback)(uint8_t *buf, size_t buf_size, void *user); -typedef void (*ChiakiTakionAVCallback)(struct chiaki_takion_av_packet_t *header, uint8_t *buf, size_t buf_size, uint8_t base_type, uint32_t key_pos, void *user); +typedef void (*ChiakiTakionAVCallback)(struct chiaki_takion_av_packet_t *packet, void *user); typedef struct chiaki_takion_connect_info_t @@ -75,13 +75,18 @@ typedef struct chiaki_takion_av_packet_t uint16_t frame_index; bool uses_nalu_info_structs; bool is_video; - uint16_t nalu_index; - uint16_t word_at_0xc; - uint16_t word_at_0xe; + uint16_t unit_index; + uint16_t units_in_frame_total; // regular + units_in_frame_additional + uint16_t units_in_frame_additional; uint32_t codec; uint16_t word_at_0x18; uint8_t adaptive_stream_index; uint8_t byte_at_0x2c; + + uint32_t key_pos; + + uint8_t *data; // not owned + size_t data_size; } ChiakiTakionAVPacket; CHIAKI_EXPORT ChiakiErrorCode chiaki_takion_connect(ChiakiTakion *takion, ChiakiTakionConnectInfo *info); @@ -94,6 +99,8 @@ static inline void chiaki_takion_set_crypt(ChiakiTakion *takion, ChiakiGKCrypt * takion->gkcrypt_remote = gkcrypt_remote; } +CHIAKI_EXPORT ChiakiErrorCode chiaki_takion_av_packet_parse(ChiakiTakionAVPacket *packet, uint8_t base_type, uint8_t *buf, size_t buf_size); + #ifdef __cplusplus } #endif diff --git a/lib/include/chiaki/videoreceiver.h b/lib/include/chiaki/videoreceiver.h index 5ed6e9b..003675d 100644 --- a/lib/include/chiaki/videoreceiver.h +++ b/lib/include/chiaki/videoreceiver.h @@ -22,6 +22,7 @@ #include "log.h" #include "video.h" #include "takion.h" +#include "frameprocessor.h" #ifdef __cplusplus extern "C" { @@ -35,6 +36,11 @@ typedef struct chiaki_video_receiver_t ChiakiLog *log; ChiakiVideoProfile profiles[CHIAKI_VIDEO_PROFILES_MAX]; size_t profiles_count; + int profile_cur; // < 1 if no profile selected yet, else index in profiles + + int32_t frame_index_cur; + int32_t frame_index_prev; + ChiakiFrameProcessor frame_processor; } ChiakiVideoReceiver; CHIAKI_EXPORT void chiaki_video_receiver_init(ChiakiVideoReceiver *video_receiver, struct chiaki_session_t *session); @@ -49,7 +55,7 @@ CHIAKI_EXPORT void chiaki_video_receiver_fini(ChiakiVideoReceiver *video_receive */ CHIAKI_EXPORT void chiaki_video_receiver_stream_info(ChiakiVideoReceiver *video_receiver, ChiakiVideoProfile *profiles, size_t profiles_count); -CHIAKI_EXPORT void chiaki_video_receiver_av_packet(ChiakiVideoReceiver *video_receiver, ChiakiTakionAVPacket *header, uint8_t *buf, size_t buf_size); +CHIAKI_EXPORT void chiaki_video_receiver_av_packet(ChiakiVideoReceiver *video_receiver, ChiakiTakionAVPacket *packet); static inline ChiakiVideoReceiver *chiaki_video_receiver_new(struct chiaki_session_t *session) { diff --git a/lib/src/frameprocessor.c b/lib/src/frameprocessor.c new file mode 100644 index 0000000..0f238a2 --- /dev/null +++ b/lib/src/frameprocessor.c @@ -0,0 +1,198 @@ +/* + * This file is part of Chiaki. + * + * Chiaki is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Chiaki is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Chiaki. If not, see . + */ + +#include + +#include + + +#define UNIT_SLOTS_MAX 256 + + +struct chiaki_frame_unit_t +{ + size_t data_size; +}; + + +CHIAKI_EXPORT void chiaki_frame_processor_init(ChiakiFrameProcessor *frame_processor, ChiakiLog *log) +{ + frame_processor->log = log; + frame_processor->frame_buf = NULL; + frame_processor->frame_buf_size = 0; + frame_processor->units_regular_expected = 0; + frame_processor->units_additional_expected = 0; + frame_processor->unit_slots = NULL; + frame_processor->unit_slots_size = 0; +} + +CHIAKI_EXPORT void chiaki_frame_processor_fini(ChiakiFrameProcessor *frame_processor) +{ + free(frame_processor->frame_buf); + free(frame_processor->unit_slots); +} + +CHIAKI_EXPORT ChiakiErrorCode chiaki_frame_processor_alloc_frame(ChiakiFrameProcessor *frame_processor, ChiakiTakionAVPacket *packet) +{ + if(packet->units_in_frame_total < packet->units_in_frame_additional) + { + CHIAKI_LOGE(frame_processor->log, "Packet has units_in_frame_total < units_in_frame_additional\n"); + return CHIAKI_ERR_INVALID_DATA; + } + + frame_processor->units_regular_expected = packet->units_in_frame_total - packet->units_in_frame_additional; + frame_processor->units_additional_expected = packet->units_in_frame_additional; + if(frame_processor->units_additional_expected < 1) + frame_processor->units_additional_expected = 1; + + frame_processor->buf_size_per_unit = packet->data_size; + if(packet->is_video && packet->unit_index < frame_processor->units_regular_expected) + { + if(packet->data_size < 2) + { + CHIAKI_LOGE(frame_processor->log, "Packet too small to read buf size extension\n"); + return CHIAKI_ERR_BUF_TOO_SMALL; + } + frame_processor->buf_size_per_unit += ntohs(((uint16_t *)packet->data)[0]); + } + + if(frame_processor->buf_size_per_unit == 0) + { + CHIAKI_LOGE(frame_processor->log, "Frame Processor doesn't handle empty units\n"); + return CHIAKI_ERR_BUF_TOO_SMALL; + } + + frame_processor->units_regular_received = 0; + frame_processor->units_additional_received = 0; + + size_t unit_slots_size_required = frame_processor->units_regular_expected + frame_processor->units_additional_expected; + if(unit_slots_size_required > UNIT_SLOTS_MAX) + { + CHIAKI_LOGE(frame_processor->log, "Packet suggests more than %u unit slots\n", UNIT_SLOTS_MAX); + return CHIAKI_ERR_INVALID_DATA; + } + if(unit_slots_size_required != frame_processor->unit_slots_size) + { + void *new_ptr = NULL; + if(frame_processor->unit_slots) + { + new_ptr = realloc(frame_processor->unit_slots, unit_slots_size_required * sizeof(ChiakiFrameUnit)); + if(!new_ptr) + free(frame_processor->unit_slots); + } + else + new_ptr = malloc(unit_slots_size_required * sizeof(ChiakiFrameUnit)); + + frame_processor->unit_slots = new_ptr; + if(!new_ptr) + { + frame_processor->unit_slots_size = 0; + return CHIAKI_ERR_MEMORY; + } + else + frame_processor->unit_slots_size = unit_slots_size_required; + } + memset(frame_processor->unit_slots, 0, frame_processor->unit_slots_size * sizeof(ChiakiFrameUnit)); + + if(frame_processor->unit_slots_size > SIZE_MAX / frame_processor->buf_size_per_unit) + return CHIAKI_ERR_OVERFLOW; + size_t frame_buf_size_required = frame_processor->unit_slots_size * frame_processor->buf_size_per_unit; + if(frame_processor->frame_buf_size < frame_buf_size_required) + { + free(frame_processor->frame_buf); + frame_processor->frame_buf = malloc(frame_buf_size_required); + if(!frame_processor->frame_buf) + { + frame_processor->frame_buf_size = 0; + return CHIAKI_ERR_MEMORY; + } + frame_processor->frame_buf_size = frame_buf_size_required; + } + memset(frame_processor->frame_buf, 0, frame_buf_size_required); + + return CHIAKI_ERR_SUCCESS; +} + +CHIAKI_EXPORT ChiakiErrorCode chiaki_frame_processor_put_unit(ChiakiFrameProcessor *frame_processor, ChiakiTakionAVPacket *packet) +{ + if(packet->unit_index > frame_processor->unit_slots_size) + { + CHIAKI_LOGE(frame_processor->log, "Packet's unit index is too high\n"); + return CHIAKI_ERR_INVALID_DATA; + } + + if(!packet->data_size) + { + CHIAKI_LOGW(frame_processor->log, "Unit is empty\n"); + return CHIAKI_ERR_INVALID_DATA; + } + + if(packet->data_size > frame_processor->buf_size_per_unit) + { + CHIAKI_LOGW(frame_processor->log, "Unit is bigger than pre-calculated size!\n"); + return CHIAKI_ERR_INVALID_DATA; + } + + ChiakiFrameUnit *unit = frame_processor->unit_slots + packet->unit_index; + if(unit->data_size) + { + CHIAKI_LOGW(frame_processor->log, "Received duplicate unit\n"); + return CHIAKI_ERR_INVALID_DATA; + } + + unit->data_size = packet->data_size; + memcpy(frame_processor->frame_buf + packet->unit_index, packet->data, packet->data_size); + + if(packet->unit_index < frame_processor->units_regular_expected) + frame_processor->units_regular_received++; + else + frame_processor->units_additional_received++; + + return CHIAKI_ERR_SUCCESS; +} + +CHIAKI_EXPORT ChiakiFrameProcessorFlushResult chiaki_frame_processor_flush(ChiakiFrameProcessor *frame_processor, uint8_t **frame, size_t *frame_size) +{ + if(frame_processor->units_regular_expected == 0) + return CHIAKI_FRAME_PROCESSOR_FLUSH_RESULT_FAILED; + + // TODO: FEC + if(frame_processor->units_regular_received < frame_processor->units_regular_expected) + return CHIAKI_FRAME_PROCESSOR_FLUSH_RESULT_FAILED; + + uint8_t *buf = malloc(frame_processor->frame_buf_size); // TODO: this should come from outside instead of mallocing all the time + if(!buf) + return CHIAKI_FRAME_PROCESSOR_FLUSH_RESULT_FAILED; + + size_t buf_size = 0; + for(size_t i=0; iunits_regular_expected; i++) + { + ChiakiFrameUnit *unit = frame_processor->unit_slots + i; + if(unit->data_size < 2) + { + CHIAKI_LOGE(frame_processor->log, "Saved unit has size < 2\n"); + continue; + } + size_t part_size = unit->data_size - 2; + memcpy(buf + buf_size, frame_processor->frame_buf + i*frame_processor->buf_size_per_unit + 2, part_size); + buf_size += part_size; + } + + *frame = buf; + *frame_size = buf_size; + return CHIAKI_FRAME_PROCESSOR_FLUSH_RESULT_SUCCESS; +} diff --git a/lib/src/nagare.c b/lib/src/nagare.c index 37629e7..e9f59dc 100644 --- a/lib/src/nagare.c +++ b/lib/src/nagare.c @@ -59,7 +59,7 @@ static ChiakiErrorCode nagare_send_disconnect(ChiakiNagare *nagare); static void nagare_takion_data_expect_bang(ChiakiNagare *nagare, uint8_t *buf, size_t buf_size); static void nagare_takion_data_expect_streaminfo(ChiakiNagare *nagare, uint8_t *buf, size_t buf_size); static ChiakiErrorCode nagare_send_streaminfo_ack(ChiakiNagare *nagare); -static void nagare_takion_av(ChiakiTakionAVPacket *header, uint8_t *buf, size_t buf_size, uint8_t base_type, uint32_t key_pos, void *user); +static void nagare_takion_av(ChiakiTakionAVPacket *packet, void *user); static ChiakiErrorCode nagare_takion_mac(uint8_t *buf, size_t buf_size, size_t key_pos, uint8_t *mac_out, void *user); @@ -550,27 +550,27 @@ static ChiakiErrorCode nagare_send_disconnect(ChiakiNagare *nagare) } -static void nagare_takion_av(ChiakiTakionAVPacket *header, uint8_t *buf, size_t buf_size, uint8_t base_type, uint32_t key_pos, void *user) +static void nagare_takion_av(ChiakiTakionAVPacket *packet, void *user) { ChiakiNagare *nagare = user; - chiaki_gkcrypt_decrypt(nagare->gkcrypt_remote, key_pos + CHIAKI_GKCRYPT_BLOCK_SIZE, buf, buf_size); + chiaki_gkcrypt_decrypt(nagare->gkcrypt_remote, packet->key_pos + CHIAKI_GKCRYPT_BLOCK_SIZE, packet->data, packet->data_size); /*CHIAKI_LOGD(nagare->log, "AV: index: %u,%u; b@0x1a: %d; is_video: %d; 0xa: %u; 0xc: %u; 0xe: %u; codec: %u; 0x18: %u; adaptive_stream: %u, 0x2c: %u\n", - header->packet_index, header->frame_index, header->byte_at_0x1a, header->is_video ? 1 : 0, header->word_at_0xa, header->word_at_0xc, header->word_at_0xe, header->codec, + header->packet_index, header->frame_index, header->byte_at_0x1a, header->is_video ? 1 : 0, header->word_at_0xa, header->units_in_frame_total, header->units_in_frame_additional, header->codec, header->word_at_0x18, header->adaptive_stream_index, header->byte_at_0x2c); chiaki_log_hexdump(nagare->log, CHIAKI_LOG_DEBUG, buf, buf_size);*/ - if(header->is_video) + if(packet->is_video) { - chiaki_video_receiver_av_packet(nagare->session->video_receiver, header, buf, buf_size); + chiaki_video_receiver_av_packet(nagare->session->video_receiver, packet); } else { - if(header->codec == 5/*buf[0] == 0xf4 && buf_size >= 0x50*/) + if(packet->codec == 5/*buf[0] == 0xf4 && buf_size >= 0x50*/) { //CHIAKI_LOGD(nagare->log, "audio!\n"); - chiaki_audio_receiver_frame_packet(nagare->session->audio_receiver, buf, 0x50); // TODO: why 0x50? this is dangerous!!! + chiaki_audio_receiver_frame_packet(nagare->session->audio_receiver, packet->data, 0x50); // TODO: why 0x50? this is dangerous!!! } else { diff --git a/lib/src/takion.c b/lib/src/takion.c index 44fd5ae..5eb5b37 100644 --- a/lib/src/takion.c +++ b/lib/src/takion.c @@ -713,80 +713,92 @@ static ChiakiErrorCode takion_recv_message_cookie_ack(ChiakiTakion *takion) } -#define AV_HEADER_SIZE_VIDEO 0x17 -#define AV_HEADER_SIZE_AUDIO 0x12 - static void takion_handle_packet_av(ChiakiTakion *takion, uint8_t base_type, uint8_t *buf, size_t buf_size) { // HHIxIIx assert(base_type == TAKION_PACKET_TYPE_VIDEO || base_type == TAKION_PACKET_TYPE_AUDIO); - size_t av_size = buf_size-1; - - ChiakiTakionAVPacket packet = {0}; - packet.is_video = base_type == TAKION_PACKET_TYPE_VIDEO; - - packet.uses_nalu_info_structs = ((buf[0] >> 4) & 1) != 0; - - size_t av_header_size = packet.is_video ? AV_HEADER_SIZE_VIDEO : AV_HEADER_SIZE_AUDIO; - if(av_size < av_header_size + 1) // TODO: compare av_size or buf_size? + ChiakiTakionAVPacket packet; + ChiakiErrorCode err = chiaki_takion_av_packet_parse(&packet, base_type, buf, buf_size); + if(err != CHIAKI_ERR_SUCCESS) { - CHIAKI_LOGE(takion->log, "Takion received AV packet smaller than av header size + 1\n"); + if(err == CHIAKI_ERR_BUF_TOO_SMALL) + CHIAKI_LOGE(takion->log, "Takion received AV packet that was too small\n"); return; } - uint8_t *av = buf+1; + if(takion->av_cb) + takion->av_cb(&packet, takion->av_cb_user); +} - packet.packet_index = ntohs(*((uint16_t *)(av + 0))); - packet.frame_index = ntohs(*((uint16_t *)(av + 2))); +#define AV_HEADER_SIZE_VIDEO 0x17 +#define AV_HEADER_SIZE_AUDIO 0x12 + +CHIAKI_EXPORT ChiakiErrorCode chiaki_takion_av_packet_parse(ChiakiTakionAVPacket *packet, uint8_t base_type, uint8_t *buf, size_t buf_size) +{ + memset(packet, 0, sizeof(ChiakiTakionAVPacket)); + if(base_type != TAKION_PACKET_TYPE_VIDEO && base_type != TAKION_PACKET_TYPE_AUDIO) + return CHIAKI_ERR_INVALID_DATA; + + if(buf_size < 1) + return CHIAKI_ERR_BUF_TOO_SMALL; + + packet->is_video = base_type == TAKION_PACKET_TYPE_VIDEO; + + packet->uses_nalu_info_structs = ((buf[0] >> 4) & 1) != 0; // TODO: is this really correct? + + uint8_t *av = buf+1; + size_t av_size = buf_size-1; + size_t av_header_size = packet->is_video ? AV_HEADER_SIZE_VIDEO : AV_HEADER_SIZE_AUDIO; + if(av_size < av_header_size + 1) + return CHIAKI_ERR_BUF_TOO_SMALL; + + packet->packet_index = ntohs(*((uint16_t *)(av + 0))); + packet->frame_index = ntohs(*((uint16_t *)(av + 2))); uint32_t dword_2 = ntohl(*((uint32_t *)(av + 4))); - if(packet.is_video) + if(packet->is_video) { - packet.nalu_index = (uint16_t)((dword_2 >> 0x15) & 0x7ff); - packet.word_at_0xc = (uint16_t)(((dword_2 >> 0xa) & 0x7ff) + 1); - packet.word_at_0xe = (uint16_t)(dword_2 & 0x3ff); + packet->unit_index = (uint16_t)((dword_2 >> 0x15) & 0x7ff); + packet->units_in_frame_total = (uint16_t)(((dword_2 >> 0xa) & 0x7ff) + 1); + packet->units_in_frame_additional = (uint16_t)(dword_2 & 0x3ff); } else { - packet.nalu_index = (uint16_t)((dword_2 >> 0x18) & 0xff); - packet.word_at_0xc = (uint16_t)(((dword_2 >> 0x10) & 0xff) + 1); - packet.word_at_0xe = (uint16_t)(dword_2 & 0xffff); + packet->unit_index = (uint16_t)((dword_2 >> 0x18) & 0xff); + packet->units_in_frame_total = (uint16_t)(((dword_2 >> 0x10) & 0xff) + 1); + packet->units_in_frame_additional = (uint16_t)(dword_2 & 0xffff); } - packet.codec = av[8]; + packet->codec = av[8]; - - uint32_t key_pos = ntohl(*((uint32_t *)(av + 0xd))); + packet->key_pos = ntohl(*((uint32_t *)(av + 0xd))); uint8_t unknown_1 = av[0x11]; - av += 0x12; - av_size -= 12; + av_size -= 0x12; - if(packet.is_video) + if(packet->is_video) { - packet.word_at_0x18 = ntohs(*((uint16_t *)(av + 0))); - packet.adaptive_stream_index = av[2] >> 5; + packet->word_at_0x18 = ntohs(*((uint16_t *)(av + 0))); + packet->adaptive_stream_index = av[2] >> 5; av += 3; av_size -= 3; } // TODO: parsing for uses_nalu_info_structs (before: packet.byte_at_0x1a) - packet.uses_nalu_info_structs = true; - if(packet.is_video) + if(packet->is_video) { - packet.byte_at_0x2c = av[0]; + packet->byte_at_0x2c = av[0]; av += 2; av_size -= 2; } - uint8_t *data = av; - size_t data_size = av_size; + packet->data = av; + packet->data_size = av_size; - if(takion->av_cb) - takion->av_cb(&packet, data, data_size, base_type, key_pos, takion->av_cb_user); -} \ No newline at end of file + return CHIAKI_ERR_SUCCESS; +} diff --git a/lib/src/videoreceiver.c b/lib/src/videoreceiver.c index 0c20220..5fce32c 100644 --- a/lib/src/videoreceiver.c +++ b/lib/src/videoreceiver.c @@ -26,12 +26,19 @@ CHIAKI_EXPORT void chiaki_video_receiver_init(ChiakiVideoReceiver *video_receive video_receiver->log = &session->log; memset(video_receiver->profiles, 0, sizeof(video_receiver->profiles)); video_receiver->profiles_count = 0; + video_receiver->profile_cur = -1; + + video_receiver->frame_index_cur = -1; + video_receiver->frame_index_prev = -1; + + chiaki_frame_processor_init(&video_receiver->frame_processor, video_receiver->log); } CHIAKI_EXPORT void chiaki_video_receiver_fini(ChiakiVideoReceiver *video_receiver) { for(size_t i=0; iprofiles_count; i++) free(video_receiver->profiles[i].header); + chiaki_frame_processor_fini(&video_receiver->frame_processor); } CHIAKI_EXPORT void chiaki_video_receiver_stream_info(ChiakiVideoReceiver *video_receiver, ChiakiVideoProfile *profiles, size_t profiles_count) @@ -52,21 +59,70 @@ CHIAKI_EXPORT void chiaki_video_receiver_stream_info(ChiakiVideoReceiver *video_ CHIAKI_LOGI(video_receiver->log, " %zu: %ux%u\n", i, profile->width, profile->height); chiaki_log_hexdump(video_receiver->log, CHIAKI_LOG_DEBUG, profile->header, profile->header_sz); } - - // TODO: should happen deferred, depending on content of av packets - if(video_receiver->session->video_sample_cb && video_receiver->profiles_count > 0) - { - video_receiver->session->video_sample_cb(video_receiver->profiles[0].header, video_receiver->profiles[0].header_sz, video_receiver->session->video_sample_cb_user); - } } -CHIAKI_EXPORT void chiaki_video_receiver_av_packet(ChiakiVideoReceiver *video_receiver, ChiakiTakionAVPacket *header, uint8_t *buf, size_t buf_size) +CHIAKI_EXPORT void chiaki_video_receiver_av_packet(ChiakiVideoReceiver *video_receiver, ChiakiTakionAVPacket *packet) { - if(video_receiver->session->video_sample_cb) + // TODO: roll over + int32_t frame_index = (int32_t)packet->frame_index; + if(frame_index < video_receiver->frame_index_cur) { - if(header->adaptive_stream_index == 0) - { - video_receiver->session->video_sample_cb(buf, buf_size, video_receiver->session->video_sample_cb_user); - } + CHIAKI_LOGW(video_receiver->log, "Video Receiver received old frame packet\n"); + return; } -} \ No newline at end of file + + if(video_receiver->profile_cur < 0 || video_receiver->profile_cur != packet->adaptive_stream_index) + { + if(packet->adaptive_stream_index >= video_receiver->profiles_count) + { + CHIAKI_LOGE(video_receiver->log, "Packet has invalid adaptive stream index %lu >= %lu\n", + (unsigned int)packet->adaptive_stream_index, + (unsigned int)video_receiver->profiles_count); + return; + } + video_receiver->profile_cur = packet->adaptive_stream_index; + + ChiakiVideoProfile *profile = video_receiver->profiles + video_receiver->profile_cur; + CHIAKI_LOGI(video_receiver->log, "Switched to profile %d, resolution: %ux%u\n", video_receiver->profile_cur, profile->width, profile->height); + if(video_receiver->session->video_sample_cb) + video_receiver->session->video_sample_cb(profile->header, profile->header_sz, video_receiver->session->video_sample_cb_user); + } + + if(frame_index > video_receiver->frame_index_cur) + { + if(video_receiver->frame_index_prev != video_receiver->frame_index_cur) + { + uint8_t *frame; + size_t frame_size; + if(chiaki_frame_processor_flush(&video_receiver->frame_processor, &frame, &frame_size) == CHIAKI_FRAME_PROCESSOR_FLUSH_RESULT_SUCCESS) + { + CHIAKI_LOGD(video_receiver->log, "Decoded frame %d\n", (int)video_receiver->frame_index_cur); + if(video_receiver->session->video_sample_cb) + video_receiver->session->video_sample_cb(frame, frame_size, video_receiver->session->video_sample_cb_user); + free(frame); + } + else + { + // TODO: fake frame? + CHIAKI_LOGW(video_receiver->log, "Failed to complete frame %d\n", (int)video_receiver->frame_index_cur); + } + + video_receiver->frame_index_prev = video_receiver->frame_index_cur; + } + + if(frame_index > video_receiver->frame_index_cur + 1 + && !(frame_index == 1 && video_receiver->frame_index_cur == -1)) // ok for frame 1 + { + CHIAKI_LOGW(video_receiver->log, "Skipped from frame %d to %d\n", (int)video_receiver->frame_index_cur, (int)frame_index); + // TODO: fake frame? + } + + video_receiver->frame_index_cur = frame_index; + CHIAKI_LOGD(video_receiver->log, "Preparing slots for frame %d\n", (int)video_receiver->frame_index_cur); + chiaki_frame_processor_alloc_frame(&video_receiver->frame_processor, packet); + } + + CHIAKI_LOGD(video_receiver->log, "Putting unit %lu of frame %d in processor\n", + (unsigned int)packet->unit_index, (int)video_receiver->frame_index_cur); + chiaki_frame_processor_put_unit(&video_receiver->frame_processor, packet); +}