Use SurfaceView on Android

android/app/src/main/cpp/video-decoder.c

@@ -26,29 +26,34 @@ ChiakiErrorCode android_chiaki_video_decoder_init(AndroidChiakiVideoDecoder *dec
 	return chiaki_mutex_init(&decoder->codec_mutex, false);
 }
 
+static void kill_decoder(AndroidChiakiVideoDecoder *decoder)
+{
+	chiaki_mutex_lock(&decoder->codec_mutex);
+	decoder->shutdown_output = true;
+	ssize_t codec_buf_index = AMediaCodec_dequeueInputBuffer(decoder->codec, 1000);
+	if(codec_buf_index >= 0)
+	{
+		CHIAKI_LOGI(decoder->log, "Video Decoder sending EOS buffer");
+		AMediaCodec_queueInputBuffer(decoder->codec, (size_t)codec_buf_index, 0, 0, decoder->timestamp_cur++, AMEDIACODEC_BUFFER_FLAG_END_OF_STREAM);
+		AMediaCodec_stop(decoder->codec);
+		chiaki_mutex_unlock(&decoder->codec_mutex);
+		chiaki_thread_join(&decoder->output_thread, NULL);
+	}
+	else
+	{
+		CHIAKI_LOGE(decoder->log, "Failed to get input buffer for shutting down Video Decoder!");
+		AMediaCodec_stop(decoder->codec);
+		chiaki_mutex_unlock(&decoder->codec_mutex);
+	}
+	AMediaCodec_delete(decoder->codec);
+	decoder->codec = NULL;
+	decoder->shutdown_output = false;
+}
+
 void android_chiaki_video_decoder_fini(AndroidChiakiVideoDecoder *decoder)
 {
 	if(decoder->codec)
-	{
-		chiaki_mutex_lock(&decoder->codec_mutex);
-		decoder->shutdown_output = true;
-		ssize_t codec_buf_index = AMediaCodec_dequeueInputBuffer(decoder->codec, -1);
-		if(codec_buf_index >= 0)
-		{
-			CHIAKI_LOGI(decoder->log, "Video Decoder sending EOS buffer");
-			AMediaCodec_queueInputBuffer(decoder->codec, (size_t)codec_buf_index, 0, 0, decoder->timestamp_cur++, AMEDIACODEC_BUFFER_FLAG_END_OF_STREAM);
-			AMediaCodec_stop(decoder->codec);
-			chiaki_mutex_unlock(&decoder->codec_mutex);
-			chiaki_thread_join(&decoder->output_thread, NULL);
-		}
-		else
-		{
-			CHIAKI_LOGE(decoder->log, "Failed to get input buffer for shutting down Video Decoder!");
-			AMediaCodec_stop(decoder->codec);
-			chiaki_mutex_unlock(&decoder->codec_mutex);
-		}
-		AMediaCodec_delete(decoder->codec);
-	}
+		kill_decoder(decoder);
 	chiaki_mutex_fini(&decoder->codec_mutex);
 }
 
@@ -56,6 +61,16 @@ void android_chiaki_video_decoder_set_surface(AndroidChiakiVideoDecoder *decoder
 {
 	chiaki_mutex_lock(&decoder->codec_mutex);
 
+	if(!surface)
+	{
+		if(decoder->codec)
+		{
+			kill_decoder(decoder);
+			CHIAKI_LOGI(decoder->log, "Decoder shut down after surface was removed");
+		}
+		return;
+	}
+
 	if(decoder->codec)
 	{
 #if __ANDROID_API__ >= 23

android/app/src/main/java/com/metallic/chiaki/lib/Chiaki.kt

@@ -91,7 +91,7 @@ private class ChiakiNative
 		@JvmStatic external fun sessionStart(ptr: Long): Int
 		@JvmStatic external fun sessionStop(ptr: Long): Int
 		@JvmStatic external fun sessionJoin(ptr: Long): Int
-		@JvmStatic external fun sessionSetSurface(ptr: Long, surface: Surface)
+		@JvmStatic external fun sessionSetSurface(ptr: Long, surface: Surface?)
 		@JvmStatic external fun sessionSetControllerState(ptr: Long, controllerState: ControllerState)
 		@JvmStatic external fun sessionSetLoginPin(ptr: Long, pin: String)
 		@JvmStatic external fun discoveryServiceCreate(result: CreateResult, options: DiscoveryServiceOptions, javaService: DiscoveryService)
@@ -350,7 +350,7 @@ class Session(connectInfo: ConnectInfo, logFile: String?, logVerbose: Boolean)
 		event(RumbleEvent(left.toUByte(), right.toUByte()))
 	}
 
-	fun setSurface(surface: Surface)
+	fun setSurface(surface: Surface?)
 	{
 		ChiakiNative.sessionSetSurface(nativePtr, surface)
 	}

android/app/src/main/java/com/metallic/chiaki/session/StreamSession.kt

@@ -28,7 +28,8 @@ class StreamSession(val connectInfo: ConnectInfo, val logManager: LogManager, va
 	private val _rumbleState = MutableLiveData<RumbleEvent>(RumbleEvent(0U, 0U))
 	val rumbleState: LiveData<RumbleEvent> get() = _rumbleState
 
-	var surfaceTexture: SurfaceTexture? = null
+	private var surfaceTexture: SurfaceTexture? = null
+	private var surface: Surface? = null
 
 	init
 	{
@@ -61,9 +62,9 @@ class StreamSession(val connectInfo: ConnectInfo, val logManager: LogManager, va
 			_state.value = StreamStateConnecting
 			session.eventCallback = this::eventCallback
 			session.start()
-			val surfaceTexture = surfaceTexture
-			if(surfaceTexture != null)
-				session.setSurface(Surface(surfaceTexture))
+			val surface = surface
+			if(surface != null)
+				session.setSurface(surface)
 			this.session = session
 		}
 		catch(e: CreateError)
@@ -92,6 +93,26 @@ class StreamSession(val connectInfo: ConnectInfo, val logManager: LogManager, va
 		}
 	}
 
+	fun attachToSurfaceView(surfaceView: SurfaceView)
+	{
+		surfaceView.holder.addCallback(object: SurfaceHolder.Callback {
+			override fun surfaceCreated(holder: SurfaceHolder) { }
+
+			override fun surfaceChanged(holder: SurfaceHolder, format: Int, width: Int, height: Int)
+			{
+				val surface = holder.surface
+				this@StreamSession.surface = surface
+				session?.setSurface(surface)
+			}
+
+			override fun surfaceDestroyed(holder: SurfaceHolder)
+			{
+				this@StreamSession.surface = null
+				session?.setSurface(null)
+			}
+		})
+	}
+
 	fun attachToTextureView(textureView: TextureView)
 	{
 		textureView.surfaceTextureListener = object: TextureView.SurfaceTextureListener {
@@ -100,6 +121,7 @@ class StreamSession(val connectInfo: ConnectInfo, val logManager: LogManager, va
 				if(surfaceTexture != null)
 					return
 				surfaceTexture = surface
+				this@StreamSession.surface = Surface(surfaceTexture)
 				session?.setSurface(Surface(surface))
 			}
 

android/app/src/main/java/com/metallic/chiaki/stream/AspectRatioFrameLayout.kt

@@ -0,0 +1,68 @@
+package com.metallic.chiaki.stream
+
+import android.content.Context
+import android.util.AttributeSet
+import android.widget.FrameLayout
+
+// see ExoPlayer's AspectRatioFrameLayout
+class AspectRatioFrameLayout @JvmOverloads constructor(context: Context, attrs: AttributeSet? = null): FrameLayout(context, attrs)
+{
+	companion object
+	{
+		private const val MAX_ASPECT_RATIO_DEFORMATION_FRACTION = 0.01f
+	}
+
+	var aspectRatio = 0f
+		set(value)
+		{
+			if(field != value)
+			{
+				field = value
+				requestLayout()
+			}
+		}
+
+	var mode: TransformMode = TransformMode.FIT
+		set(value)
+		{
+			if(field != value)
+			{
+				field = value
+				requestLayout()
+			}
+		}
+
+	override fun onMeasure(widthMeasureSpec: Int, heightMeasureSpec: Int)
+	{
+		super.onMeasure(widthMeasureSpec, heightMeasureSpec)
+		if(aspectRatio <= 0)
+		{
+			// Aspect ratio not set.
+			return
+		}
+		var width = measuredWidth
+		var height = measuredHeight
+		val viewAspectRatio = width.toFloat() / height
+		val aspectDeformation = aspectRatio / viewAspectRatio - 1
+		if(Math.abs(aspectDeformation) <= MAX_ASPECT_RATIO_DEFORMATION_FRACTION)
+			return
+		when(mode)
+		{
+			TransformMode.ZOOM -> 
+				if(aspectDeformation > 0)
+					width = (height * aspectRatio).toInt()
+				else
+					height = (width / aspectRatio).toInt()
+			TransformMode.FIT ->
+				if(aspectDeformation > 0)
+					height = (width / aspectRatio).toInt()
+				else
+					width = (height * aspectRatio).toInt()
+			TransformMode.STRETCH -> {}
+		}
+		super.onMeasure(
+			MeasureSpec.makeMeasureSpec(width, MeasureSpec.EXACTLY),
+			MeasureSpec.makeMeasureSpec(height, MeasureSpec.EXACTLY)
+		)
+	}
+}

android/app/src/main/java/com/metallic/chiaki/stream/StreamActivity.kt

@@ -7,12 +7,12 @@ import android.animation.AnimatorListenerAdapter
 import android.app.AlertDialog
 import android.graphics.Matrix
 import android.os.*
-import android.view.KeyEvent
-import android.view.MotionEvent
-import android.view.TextureView
-import android.view.View
+import android.transition.TransitionManager
+import android.view.*
 import android.widget.EditText
 import androidx.appcompat.app.AppCompatActivity
+import androidx.constraintlayout.widget.ConstraintLayout
+import androidx.constraintlayout.widget.ConstraintSet
 import androidx.core.view.isGone
 import androidx.core.view.isVisible
 import androidx.fragment.app.Fragment
@@ -44,6 +44,7 @@ class StreamActivity : AppCompatActivity(), View.OnSystemUiVisibilityChangeListe
 
 	private lateinit var viewModel: StreamViewModel
 	private val uiVisibilityHandler = Handler()
+	private val streamView: View get() = surfaceView
 
 	override fun onCreate(savedInstanceState: Bundle?)
 	{
@@ -94,15 +95,17 @@ class StreamActivity : AppCompatActivity(), View.OnSystemUiVisibilityChangeListe
 			if (displayModeToggle.checkedButtonId == -1)
 				displayModeToggle.check(checkedId)
 
-			adjustTextureViewAspect()
+			adjustStreamViewAspect()
 			showOverlay()
 		}
 
-		viewModel.session.attachToTextureView(textureView)
+		//viewModel.session.attachToTextureView(textureView)
+		viewModel.session.attachToSurfaceView(surfaceView)
 		viewModel.session.state.observe(this, Observer { this.stateChanged(it) })
-		textureView.addOnLayoutChangeListener { _, _, _, _, _, _, _, _, _ ->
-			adjustTextureViewAspect()
-		}
+		/*streamView.addOnLayoutChangeListener { _, _, _, _, _, _, _, _, _ ->
+			adjustStreamViewAspect()
+		}*/
+		adjustStreamViewAspect()
 
 		if(Preferences(this).rumbleEnabled)
 		{
@@ -306,73 +309,84 @@ class StreamActivity : AppCompatActivity(), View.OnSystemUiVisibilityChangeListe
 		}
 	}
 
-	private fun adjustTextureViewAspect()
+	private fun adjustTextureViewAspect(textureView: TextureView)
 	{
-		val displayInfo = DisplayInfo(viewModel.session.connectInfo.videoProfile, textureView)
-		val resolution = displayInfo.computeResolutionFor(displayModeToggle.checkedButtonId)
-
+		val trans = TextureViewTransform(viewModel.session.connectInfo.videoProfile, textureView)
+		val resolution = trans.resolutionFor(TransformMode.fromButton(displayModeToggle.checkedButtonId))
 		Matrix().also {
 			textureView.getTransform(it)
-			it.setScale(resolution.width / displayInfo.viewWidth, resolution.height / displayInfo.viewHeight)
-			it.postTranslate((displayInfo.viewWidth - resolution.width) * 0.5f, (displayInfo.viewHeight - resolution.height) * 0.5f)
+			it.setScale(resolution.width / trans.viewWidth, resolution.height / trans.viewHeight)
+			it.postTranslate((trans.viewWidth - resolution.width) * 0.5f, (trans.viewHeight - resolution.height) * 0.5f)
 			textureView.setTransform(it)
 		}
 	}
 
+	private fun adjustSurfaceViewAspect()
+	{
+		val videoProfile = viewModel.session.connectInfo.videoProfile
+		aspectRatioLayout.aspectRatio = videoProfile.width.toFloat() / videoProfile.height.toFloat()
+		aspectRatioLayout.mode = TransformMode.fromButton(displayModeToggle.checkedButtonId)
+	}
+
+	private fun adjustStreamViewAspect() = adjustSurfaceViewAspect()
+
 	override fun dispatchKeyEvent(event: KeyEvent) = viewModel.input.dispatchKeyEvent(event) || super.dispatchKeyEvent(event)
 	override fun onGenericMotionEvent(event: MotionEvent) = viewModel.input.onGenericMotionEvent(event) || super.onGenericMotionEvent(event)
 }
 
-
-class DisplayInfo constructor(val videoProfile: ConnectVideoProfile, val textureView: TextureView)
+enum class TransformMode
 {
-	val contentWidth : Float get() = videoProfile.width.toFloat()
-	val contentHeight : Float get() = videoProfile.height.toFloat()
+	FIT,
+	STRETCH,
+	ZOOM;
+
+	companion object
+	{
+		fun fromButton(displayModeButtonId: Int)
+			= when (displayModeButtonId)
+			{
+				R.id.display_mode_stretch_button -> STRETCH
+				R.id.display_mode_zoom_button -> ZOOM
+				else -> FIT
+			}
+	}
+}
+
+class TextureViewTransform(private val videoProfile: ConnectVideoProfile, private val textureView: TextureView)
+{
+	private val contentWidth : Float get() = videoProfile.width.toFloat()
+	private val contentHeight : Float get() = videoProfile.height.toFloat()
 	val viewWidth : Float get() = textureView.width.toFloat()
 	val viewHeight : Float get() = textureView.height.toFloat()
-	val contentAspect : Float get() =  contentHeight / contentWidth
+	private val contentAspect : Float get() =  contentHeight / contentWidth
 
-	fun computeResolutionFor(displayModeButtonId: Int) : Resolution
-	{
-		when (displayModeButtonId)
+	fun resolutionFor(mode: TransformMode): Resolution
+		= when(mode)
 		{
-			R.id.display_mode_stretch_button -> return computeStrechedResolution()
-			R.id.display_mode_zoom_button -> return computeZoomedResolution()
-			else -> return computeNormalResolution()
+			TransformMode.STRETCH -> strechedResolution
+			TransformMode.ZOOM -> zoomedResolution
+			TransformMode.FIT -> normalResolution
 		}
-	}
 
-	private fun computeStrechedResolution(): Resolution
-	{
-		return Resolution(viewWidth, viewHeight)
-	}
+	private val strechedResolution get() = Resolution(viewWidth, viewHeight)
 
-	private fun computeZoomedResolution(): Resolution
-	{
-		if (viewHeight > viewWidth * contentAspect)
+	private val zoomedResolution get() =
+		if(viewHeight > viewWidth * contentAspect)
 		{
 			val zoomFactor = viewHeight / contentHeight
-			return Resolution(contentWidth * zoomFactor, viewHeight)
+			Resolution(contentWidth * zoomFactor, viewHeight)
 		}
 		else
 		{
 			val zoomFactor = viewWidth / contentWidth
-			return Resolution(viewWidth, contentHeight * zoomFactor)
+			Resolution(viewWidth, contentHeight * zoomFactor)
 		}
-	}
 
-	private fun computeNormalResolution(): Resolution
-	{
-		if (viewHeight > viewWidth * contentAspect)
-		{
-			return Resolution(viewWidth, viewWidth * contentAspect)
-		}
+	private val normalResolution get() =
+		if(viewHeight > viewWidth * contentAspect)
+			Resolution(viewWidth, viewWidth * contentAspect)
 		else
-		{
-			return Resolution(viewHeight / contentAspect, viewHeight)
-		}
-	}
-
+			Resolution(viewHeight / contentAspect, viewHeight)
 }
 
 

android/app/src/main/res/layout/activity_stream.xml

@@ -1,16 +1,24 @@
 <?xml version="1.0" encoding="utf-8"?>
-<FrameLayout xmlns:android="http://schemas.android.com/apk/res/android"
+<FrameLayout
+    xmlns:android="http://schemas.android.com/apk/res/android"
     xmlns:tools="http://schemas.android.com/tools"
+    xmlns:app="http://schemas.android.com/apk/res-auto"
+    android:id="@+id/mainStreamLayout"
     android:layout_width="match_parent"
     android:layout_height="match_parent"
-    xmlns:app="http://schemas.android.com/apk/res-auto"
     tools:context=".stream.StreamActivity"
     android:keepScreenOn="true">
 
-    <TextureView
-        android:id="@+id/textureView"
+    <com.metallic.chiaki.stream.AspectRatioFrameLayout
+        android:id="@+id/aspectRatioLayout"
         android:layout_width="match_parent"
-        android:layout_height="match_parent"/>
+        android:layout_height="match_parent"
+        android:layout_gravity="center">
+        <SurfaceView
+            android:id="@+id/surfaceView"
+            android:layout_width="wrap_content"
+            android:layout_height="wrap_content"/>
+    </com.metallic.chiaki.stream.AspectRatioFrameLayout>
 
     <ProgressBar
         android:id="@+id/progressBar"
@@ -34,7 +42,6 @@
         android:id="@+id/overlay"
         android:layout_width="match_parent"
         android:layout_height="match_parent"
-        android:layout_gravity="bottom"
         android:fitsSystemWindows="true">
 
         <androidx.constraintlayout.widget.ConstraintLayout