Basic Video Output on Android

android/app/CMakeLists.txt

@@ -8,4 +8,6 @@ add_library(chiaki-jni SHARED
 target_link_libraries(chiaki-jni chiaki-lib)
 
 find_library(ANDROID_LIB_LOG log)
-target_link_libraries(chiaki-jni "${ANDROID_LIB_LOG}")
+find_library(ANDROID_LIB_MEDIANDK mediandk)
+find_library(ANDROID_LIB_ANDROID android)
+target_link_libraries(chiaki-jni "${ANDROID_LIB_LOG}" "${ANDROID_LIB_MEDIANDK}" "${ANDROID_LIB_ANDROID}")

android/app/src/main/cpp/chiaki-jni.c

@@ -119,6 +119,7 @@ static JNIEnv *attach_thread_jni()
 		return env;
 
 	CHIAKI_LOGE(&global_log, "Failed to get JNIEnv from JavaVM or attach");
+	return NULL;
 }
 
 static void android_chiaki_event_cb(ChiakiEvent *event, void *user)
@@ -270,3 +271,9 @@ JNIEXPORT jint JNICALL Java_com_metallic_chiaki_lib_ChiakiNative_sessionJoin(JNI
 	CHIAKI_LOGI(&global_log, "Join JNI Session");
 	return chiaki_session_join(&session->session);
 }
+
+JNIEXPORT void JNICALL Java_com_metallic_chiaki_lib_ChiakiNative_sessionSetSurface(JNIEnv *env, jobject obj, jlong ptr, jobject surface)
+{
+	AndroidChiakiSession *session = (AndroidChiakiSession *)ptr;
+	android_chiaki_video_decoder_set_surface(&session->video_decoder, env, surface);
+}

android/app/src/main/cpp/video-decoder.c

@@ -17,12 +17,21 @@
 
 #include "video-decoder.h"
 
+#include <jni.h>
+
 #include <media/NdkMediaCodec.h>
+#include <media/NdkMediaFormat.h>
+#include <android/native_window_jni.h>
+
+#include <string.h>
+
+static void android_chiaki_video_decoder_flush(AndroidChiakiVideoDecoder *decoder);
 
 ChiakiErrorCode android_chiaki_video_decoder_init(AndroidChiakiVideoDecoder *decoder, ChiakiLog *log)
 {
 	decoder->log = log;
-	decoder->video_codec = NULL;
+	decoder->codec = NULL;
+	decoder->timestamp_cur = 0;
 	return chiaki_mutex_init(&decoder->mutex, false);
 }
 
@@ -31,8 +40,96 @@ void android_chiaki_video_decoder_fini(AndroidChiakiVideoDecoder *decoder)
 	chiaki_mutex_fini(&decoder->mutex);
 }
 
+void android_chiaki_video_decoder_set_surface(AndroidChiakiVideoDecoder *decoder, JNIEnv *env, jobject surface)
+{
+	chiaki_mutex_lock(&decoder->mutex);
+
+	if(decoder->codec)
+	{
+		// TODO: destroy old
+		CHIAKI_LOGE(decoder->log, "Video Decoder already initialized");
+		goto beach;
+	}
+
+	decoder->window = ANativeWindow_fromSurface(env, surface);
+
+	const char *mime = "video/avc";
+
+	decoder->codec = AMediaCodec_createDecoderByType(mime);
+	if(!decoder->codec)
+	{
+		CHIAKI_LOGE(decoder->log, "Failed to create AMediaCodec for mime type %s", mime);
+		ANativeWindow_release(decoder->window);
+		decoder->window = NULL;
+		goto beach;
+	}
+
+	AMediaFormat *format = AMediaFormat_new();
+	AMediaFormat_setString(format, AMEDIAFORMAT_KEY_MIME, mime);
+	AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_WIDTH, 1280); // TODO: correct values
+	AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_HEIGHT, 720);
+
+	AMediaCodec_configure(decoder->codec, format, decoder->window, NULL, 0);
+	AMediaCodec_start(decoder->codec);
+
+	AMediaFormat_delete(format);
+
+beach:
+	chiaki_mutex_unlock(&decoder->mutex);
+}
+
 void android_chiaki_video_decoder_video_sample(uint8_t *buf, size_t buf_size, void *user)
 {
 	AndroidChiakiVideoDecoder *decoder = user;
+	chiaki_mutex_lock(&decoder->mutex);
+
+	if(!decoder->codec)
+	{
+		CHIAKI_LOGE(decoder->log, "Received video data, but decoder is not initialized!");
+		goto beach;
+	}
+
+	ssize_t codec_buf_index;
+
 	CHIAKI_LOGD(decoder->log, "Got video sample of size %zu", buf_size);
+
+	while(buf_size > 0)
+	{
+		codec_buf_index = AMediaCodec_dequeueInputBuffer(decoder->codec, 100); // TODO: lower timeout?
+		if(codec_buf_index < 0)
+		{
+			// TODO: handle better
+			CHIAKI_LOGE(decoder->log, "Failed to get input buffer");
+			goto beach;
+		}
+
+		size_t codec_buf_size;
+		uint8_t *codec_buf = AMediaCodec_getInputBuffer(decoder->codec, (size_t)codec_buf_index, &codec_buf_size);
+		size_t codec_sample_size = buf_size;
+		if(codec_sample_size > codec_buf_size)
+		{
+			CHIAKI_LOGD(decoder->log, "Sample is bigger than buffer, splitting");
+			codec_sample_size = codec_buf_size;
+		}
+		memcpy(codec_buf, buf, codec_sample_size);
+		AMediaCodec_queueInputBuffer(decoder->codec, (size_t)codec_buf_index, 0, codec_sample_size, decoder->timestamp_cur++, 0); // timestamp just raised by 1 for maximum realtime
+		buf += codec_sample_size;
+		buf_size -= codec_sample_size;
+
+		AMediaCodecBufferInfo info;
+		ssize_t status = AMediaCodec_dequeueOutputBuffer(decoder->codec, &info, 0);
+		if(status >= 0)
+		{
+			AMediaCodec_releaseOutputBuffer(decoder->codec, (size_t)status, info.size != 0);
+		}
+	}
+
+beach:
+	android_chiaki_video_decoder_flush(decoder);
+	chiaki_mutex_unlock(&decoder->mutex);
+}
+
+static void android_chiaki_video_decoder_flush(AndroidChiakiVideoDecoder *decoder)
+{
+	// decoder->mutex must be already locked
 }

android/app/src/main/cpp/video-decoder.h

@@ -18,20 +18,26 @@
 #ifndef CHIAKI_JNI_VIDEO_DECODER_H
 #define CHIAKI_JNI_VIDEO_DECODER_H
 
+#include <jni.h>
+
 #include <chiaki/thread.h>
 #include <chiaki/log.h>
 
 typedef struct AMediaCodec AMediaCodec;
+typedef struct ANativeWindow ANativeWindow;
 
 typedef struct android_chiaki_video_decoder_t
 {
 	ChiakiLog *log;
 	ChiakiMutex mutex;
-	AMediaCodec *video_codec;
+	AMediaCodec *codec;
+	ANativeWindow *window;
+	uint64_t timestamp_cur;
 } AndroidChiakiVideoDecoder;
 
 ChiakiErrorCode android_chiaki_video_decoder_init(AndroidChiakiVideoDecoder *decoder, ChiakiLog *log);
 void android_chiaki_video_decoder_fini(AndroidChiakiVideoDecoder *decoder);
+void android_chiaki_video_decoder_set_surface(AndroidChiakiVideoDecoder *decoder, JNIEnv *env, jobject surface);
 void android_chiaki_video_decoder_video_sample(uint8_t *buf, size_t buf_size, void *user);
 
 #endif

android/app/src/main/java/com/metallic/chiaki/lib/Chiaki.kt

@@ -2,6 +2,7 @@ package com.metallic.chiaki.lib
 
 import android.os.Parcelable
 import android.util.Log
+import android.view.Surface
 import kotlinx.android.parcel.Parcelize
 import java.lang.Exception
 
@@ -37,6 +38,7 @@ class ChiakiNative
 		@JvmStatic external fun sessionStart(ptr: Long): Int
 		@JvmStatic external fun sessionStop(ptr: Long): Int
 		@JvmStatic external fun sessionJoin(ptr: Long): Int
+		@JvmStatic external fun sessionSetSurface(ptr: Long, surface: Surface)
 	}
 }
 
@@ -103,4 +105,9 @@ class Session(connectInfo: ConnectInfo)
 	{
 		event(QuitEvent(QuitReason(reasonValue), reasonString))
 	}
+
+	fun setSurface(surface: Surface)
+	{
+		ChiakiNative.sessionSetSurface(nativePtr, surface)
+	}
 }

android/app/src/main/java/com/metallic/chiaki/stream/StreamActivity.kt

@@ -2,6 +2,7 @@ package com.metallic.chiaki.stream
 
 import android.os.Bundle
 import android.util.Log
+import android.view.SurfaceHolder
 import androidx.appcompat.app.AppCompatActivity
 import androidx.lifecycle.Observer
 import androidx.lifecycle.ViewModelProvider
@@ -38,6 +39,21 @@ class StreamActivity : AppCompatActivity()
 			viewModel.init(connectInfo)
 		}
 
+		surfaceView.holder.addCallback(object: SurfaceHolder.Callback {
+			override fun surfaceChanged(holder: SurfaceHolder, format: Int, width: Int, height: Int)
+			{
+			}
+
+			override fun surfaceDestroyed(holder: SurfaceHolder)
+			{
+			}
+
+			override fun surfaceCreated(holder: SurfaceHolder)
+			{
+				viewModel.session?.setSurface(holder.surface)
+			}
+		})
+
 		viewModel.state.observe(this, Observer {
 			stateTextView.text = "$it"
 		})

android/app/src/main/java/com/metallic/chiaki/stream/StreamViewModel.kt

@@ -16,7 +16,8 @@ class StreamViewModel: ViewModel()
 	var isInitialized = false
 		private set(value) { field = value}
 
-	private var session: Session? = null
+	var session: Session? = null
+		private set(value) { field = value }
 
 	private val _state = MutableLiveData<StreamState>(StreamStateIdle)
 	val state: LiveData<StreamState> get() = _state