diff --git a/android/app/src/main/cpp/video-decoder.c b/android/app/src/main/cpp/video-decoder.c index d57623d..1146ad8 100644 --- a/android/app/src/main/cpp/video-decoder.c +++ b/android/app/src/main/cpp/video-decoder.c @@ -26,29 +26,34 @@ ChiakiErrorCode android_chiaki_video_decoder_init(AndroidChiakiVideoDecoder *dec return chiaki_mutex_init(&decoder->codec_mutex, false); } +static void kill_decoder(AndroidChiakiVideoDecoder *decoder) +{ + chiaki_mutex_lock(&decoder->codec_mutex); + decoder->shutdown_output = true; + ssize_t codec_buf_index = AMediaCodec_dequeueInputBuffer(decoder->codec, 1000); + if(codec_buf_index >= 0) + { + CHIAKI_LOGI(decoder->log, "Video Decoder sending EOS buffer"); + AMediaCodec_queueInputBuffer(decoder->codec, (size_t)codec_buf_index, 0, 0, decoder->timestamp_cur++, AMEDIACODEC_BUFFER_FLAG_END_OF_STREAM); + AMediaCodec_stop(decoder->codec); + chiaki_mutex_unlock(&decoder->codec_mutex); + chiaki_thread_join(&decoder->output_thread, NULL); + } + else + { + CHIAKI_LOGE(decoder->log, "Failed to get input buffer for shutting down Video Decoder!"); + AMediaCodec_stop(decoder->codec); + chiaki_mutex_unlock(&decoder->codec_mutex); + } + AMediaCodec_delete(decoder->codec); + decoder->codec = NULL; + decoder->shutdown_output = false; +} + void android_chiaki_video_decoder_fini(AndroidChiakiVideoDecoder *decoder) { if(decoder->codec) - { - chiaki_mutex_lock(&decoder->codec_mutex); - decoder->shutdown_output = true; - ssize_t codec_buf_index = AMediaCodec_dequeueInputBuffer(decoder->codec, -1); - if(codec_buf_index >= 0) - { - CHIAKI_LOGI(decoder->log, "Video Decoder sending EOS buffer"); - AMediaCodec_queueInputBuffer(decoder->codec, (size_t)codec_buf_index, 0, 0, decoder->timestamp_cur++, AMEDIACODEC_BUFFER_FLAG_END_OF_STREAM); - AMediaCodec_stop(decoder->codec); - chiaki_mutex_unlock(&decoder->codec_mutex); - chiaki_thread_join(&decoder->output_thread, NULL); - } - else - { - CHIAKI_LOGE(decoder->log, "Failed to get input buffer for shutting down Video Decoder!"); - AMediaCodec_stop(decoder->codec); - chiaki_mutex_unlock(&decoder->codec_mutex); - } - AMediaCodec_delete(decoder->codec); - } + kill_decoder(decoder); chiaki_mutex_fini(&decoder->codec_mutex); } @@ -56,6 +61,16 @@ void android_chiaki_video_decoder_set_surface(AndroidChiakiVideoDecoder *decoder { chiaki_mutex_lock(&decoder->codec_mutex); + if(!surface) + { + if(decoder->codec) + { + kill_decoder(decoder); + CHIAKI_LOGI(decoder->log, "Decoder shut down after surface was removed"); + } + return; + } + if(decoder->codec) { #if __ANDROID_API__ >= 23 diff --git a/android/app/src/main/java/com/metallic/chiaki/lib/Chiaki.kt b/android/app/src/main/java/com/metallic/chiaki/lib/Chiaki.kt index 023da90..b8d7204 100644 --- a/android/app/src/main/java/com/metallic/chiaki/lib/Chiaki.kt +++ b/android/app/src/main/java/com/metallic/chiaki/lib/Chiaki.kt @@ -91,7 +91,7 @@ private class ChiakiNative @JvmStatic external fun sessionStart(ptr: Long): Int @JvmStatic external fun sessionStop(ptr: Long): Int @JvmStatic external fun sessionJoin(ptr: Long): Int - @JvmStatic external fun sessionSetSurface(ptr: Long, surface: Surface) + @JvmStatic external fun sessionSetSurface(ptr: Long, surface: Surface?) @JvmStatic external fun sessionSetControllerState(ptr: Long, controllerState: ControllerState) @JvmStatic external fun sessionSetLoginPin(ptr: Long, pin: String) @JvmStatic external fun discoveryServiceCreate(result: CreateResult, options: DiscoveryServiceOptions, javaService: DiscoveryService) @@ -350,7 +350,7 @@ class Session(connectInfo: ConnectInfo, logFile: String?, logVerbose: Boolean) event(RumbleEvent(left.toUByte(), right.toUByte())) } - fun setSurface(surface: Surface) + fun setSurface(surface: Surface?) { ChiakiNative.sessionSetSurface(nativePtr, surface) } diff --git a/android/app/src/main/java/com/metallic/chiaki/session/StreamSession.kt b/android/app/src/main/java/com/metallic/chiaki/session/StreamSession.kt index d7f127f..dd32a40 100644 --- a/android/app/src/main/java/com/metallic/chiaki/session/StreamSession.kt +++ b/android/app/src/main/java/com/metallic/chiaki/session/StreamSession.kt @@ -28,7 +28,8 @@ class StreamSession(val connectInfo: ConnectInfo, val logManager: LogManager, va private val _rumbleState = MutableLiveData(RumbleEvent(0U, 0U)) val rumbleState: LiveData get() = _rumbleState - var surfaceTexture: SurfaceTexture? = null + private var surfaceTexture: SurfaceTexture? = null + private var surface: Surface? = null init { @@ -61,9 +62,9 @@ class StreamSession(val connectInfo: ConnectInfo, val logManager: LogManager, va _state.value = StreamStateConnecting session.eventCallback = this::eventCallback session.start() - val surfaceTexture = surfaceTexture - if(surfaceTexture != null) - session.setSurface(Surface(surfaceTexture)) + val surface = surface + if(surface != null) + session.setSurface(surface) this.session = session } catch(e: CreateError) @@ -92,6 +93,26 @@ class StreamSession(val connectInfo: ConnectInfo, val logManager: LogManager, va } } + fun attachToSurfaceView(surfaceView: SurfaceView) + { + surfaceView.holder.addCallback(object: SurfaceHolder.Callback { + override fun surfaceCreated(holder: SurfaceHolder) { } + + override fun surfaceChanged(holder: SurfaceHolder, format: Int, width: Int, height: Int) + { + val surface = holder.surface + this@StreamSession.surface = surface + session?.setSurface(surface) + } + + override fun surfaceDestroyed(holder: SurfaceHolder) + { + this@StreamSession.surface = null + session?.setSurface(null) + } + }) + } + fun attachToTextureView(textureView: TextureView) { textureView.surfaceTextureListener = object: TextureView.SurfaceTextureListener { @@ -100,6 +121,7 @@ class StreamSession(val connectInfo: ConnectInfo, val logManager: LogManager, va if(surfaceTexture != null) return surfaceTexture = surface + this@StreamSession.surface = Surface(surfaceTexture) session?.setSurface(Surface(surface)) } diff --git a/android/app/src/main/java/com/metallic/chiaki/stream/AspectRatioFrameLayout.kt b/android/app/src/main/java/com/metallic/chiaki/stream/AspectRatioFrameLayout.kt new file mode 100644 index 0000000..5394f02 --- /dev/null +++ b/android/app/src/main/java/com/metallic/chiaki/stream/AspectRatioFrameLayout.kt @@ -0,0 +1,68 @@ +package com.metallic.chiaki.stream + +import android.content.Context +import android.util.AttributeSet +import android.widget.FrameLayout + +// see ExoPlayer's AspectRatioFrameLayout +class AspectRatioFrameLayout @JvmOverloads constructor(context: Context, attrs: AttributeSet? = null): FrameLayout(context, attrs) +{ + companion object + { + private const val MAX_ASPECT_RATIO_DEFORMATION_FRACTION = 0.01f + } + + var aspectRatio = 0f + set(value) + { + if(field != value) + { + field = value + requestLayout() + } + } + + var mode: TransformMode = TransformMode.FIT + set(value) + { + if(field != value) + { + field = value + requestLayout() + } + } + + override fun onMeasure(widthMeasureSpec: Int, heightMeasureSpec: Int) + { + super.onMeasure(widthMeasureSpec, heightMeasureSpec) + if(aspectRatio <= 0) + { + // Aspect ratio not set. + return + } + var width = measuredWidth + var height = measuredHeight + val viewAspectRatio = width.toFloat() / height + val aspectDeformation = aspectRatio / viewAspectRatio - 1 + if(Math.abs(aspectDeformation) <= MAX_ASPECT_RATIO_DEFORMATION_FRACTION) + return + when(mode) + { + TransformMode.ZOOM -> + if(aspectDeformation > 0) + width = (height * aspectRatio).toInt() + else + height = (width / aspectRatio).toInt() + TransformMode.FIT -> + if(aspectDeformation > 0) + height = (width / aspectRatio).toInt() + else + width = (height * aspectRatio).toInt() + TransformMode.STRETCH -> {} + } + super.onMeasure( + MeasureSpec.makeMeasureSpec(width, MeasureSpec.EXACTLY), + MeasureSpec.makeMeasureSpec(height, MeasureSpec.EXACTLY) + ) + } +} diff --git a/android/app/src/main/java/com/metallic/chiaki/stream/StreamActivity.kt b/android/app/src/main/java/com/metallic/chiaki/stream/StreamActivity.kt index a25772d..32b9fd3 100644 --- a/android/app/src/main/java/com/metallic/chiaki/stream/StreamActivity.kt +++ b/android/app/src/main/java/com/metallic/chiaki/stream/StreamActivity.kt @@ -7,12 +7,12 @@ import android.animation.AnimatorListenerAdapter import android.app.AlertDialog import android.graphics.Matrix import android.os.* -import android.view.KeyEvent -import android.view.MotionEvent -import android.view.TextureView -import android.view.View +import android.transition.TransitionManager +import android.view.* import android.widget.EditText import androidx.appcompat.app.AppCompatActivity +import androidx.constraintlayout.widget.ConstraintLayout +import androidx.constraintlayout.widget.ConstraintSet import androidx.core.view.isGone import androidx.core.view.isVisible import androidx.fragment.app.Fragment @@ -44,6 +44,7 @@ class StreamActivity : AppCompatActivity(), View.OnSystemUiVisibilityChangeListe private lateinit var viewModel: StreamViewModel private val uiVisibilityHandler = Handler() + private val streamView: View get() = surfaceView override fun onCreate(savedInstanceState: Bundle?) { @@ -94,15 +95,17 @@ class StreamActivity : AppCompatActivity(), View.OnSystemUiVisibilityChangeListe if (displayModeToggle.checkedButtonId == -1) displayModeToggle.check(checkedId) - adjustTextureViewAspect() + adjustStreamViewAspect() showOverlay() } - viewModel.session.attachToTextureView(textureView) + //viewModel.session.attachToTextureView(textureView) + viewModel.session.attachToSurfaceView(surfaceView) viewModel.session.state.observe(this, Observer { this.stateChanged(it) }) - textureView.addOnLayoutChangeListener { _, _, _, _, _, _, _, _, _ -> - adjustTextureViewAspect() - } + /*streamView.addOnLayoutChangeListener { _, _, _, _, _, _, _, _, _ -> + adjustStreamViewAspect() + }*/ + adjustStreamViewAspect() if(Preferences(this).rumbleEnabled) { @@ -306,73 +309,84 @@ class StreamActivity : AppCompatActivity(), View.OnSystemUiVisibilityChangeListe } } - private fun adjustTextureViewAspect() + private fun adjustTextureViewAspect(textureView: TextureView) { - val displayInfo = DisplayInfo(viewModel.session.connectInfo.videoProfile, textureView) - val resolution = displayInfo.computeResolutionFor(displayModeToggle.checkedButtonId) - + val trans = TextureViewTransform(viewModel.session.connectInfo.videoProfile, textureView) + val resolution = trans.resolutionFor(TransformMode.fromButton(displayModeToggle.checkedButtonId)) Matrix().also { textureView.getTransform(it) - it.setScale(resolution.width / displayInfo.viewWidth, resolution.height / displayInfo.viewHeight) - it.postTranslate((displayInfo.viewWidth - resolution.width) * 0.5f, (displayInfo.viewHeight - resolution.height) * 0.5f) + it.setScale(resolution.width / trans.viewWidth, resolution.height / trans.viewHeight) + it.postTranslate((trans.viewWidth - resolution.width) * 0.5f, (trans.viewHeight - resolution.height) * 0.5f) textureView.setTransform(it) } } + private fun adjustSurfaceViewAspect() + { + val videoProfile = viewModel.session.connectInfo.videoProfile + aspectRatioLayout.aspectRatio = videoProfile.width.toFloat() / videoProfile.height.toFloat() + aspectRatioLayout.mode = TransformMode.fromButton(displayModeToggle.checkedButtonId) + } + + private fun adjustStreamViewAspect() = adjustSurfaceViewAspect() + override fun dispatchKeyEvent(event: KeyEvent) = viewModel.input.dispatchKeyEvent(event) || super.dispatchKeyEvent(event) override fun onGenericMotionEvent(event: MotionEvent) = viewModel.input.onGenericMotionEvent(event) || super.onGenericMotionEvent(event) } - -class DisplayInfo constructor(val videoProfile: ConnectVideoProfile, val textureView: TextureView) +enum class TransformMode { - val contentWidth : Float get() = videoProfile.width.toFloat() - val contentHeight : Float get() = videoProfile.height.toFloat() + FIT, + STRETCH, + ZOOM; + + companion object + { + fun fromButton(displayModeButtonId: Int) + = when (displayModeButtonId) + { + R.id.display_mode_stretch_button -> STRETCH + R.id.display_mode_zoom_button -> ZOOM + else -> FIT + } + } +} + +class TextureViewTransform(private val videoProfile: ConnectVideoProfile, private val textureView: TextureView) +{ + private val contentWidth : Float get() = videoProfile.width.toFloat() + private val contentHeight : Float get() = videoProfile.height.toFloat() val viewWidth : Float get() = textureView.width.toFloat() val viewHeight : Float get() = textureView.height.toFloat() - val contentAspect : Float get() = contentHeight / contentWidth + private val contentAspect : Float get() = contentHeight / contentWidth - fun computeResolutionFor(displayModeButtonId: Int) : Resolution - { - when (displayModeButtonId) + fun resolutionFor(mode: TransformMode): Resolution + = when(mode) { - R.id.display_mode_stretch_button -> return computeStrechedResolution() - R.id.display_mode_zoom_button -> return computeZoomedResolution() - else -> return computeNormalResolution() + TransformMode.STRETCH -> strechedResolution + TransformMode.ZOOM -> zoomedResolution + TransformMode.FIT -> normalResolution } - } - private fun computeStrechedResolution(): Resolution - { - return Resolution(viewWidth, viewHeight) - } + private val strechedResolution get() = Resolution(viewWidth, viewHeight) - private fun computeZoomedResolution(): Resolution - { - if (viewHeight > viewWidth * contentAspect) + private val zoomedResolution get() = + if(viewHeight > viewWidth * contentAspect) { val zoomFactor = viewHeight / contentHeight - return Resolution(contentWidth * zoomFactor, viewHeight) + Resolution(contentWidth * zoomFactor, viewHeight) } else { val zoomFactor = viewWidth / contentWidth - return Resolution(viewWidth, contentHeight * zoomFactor) + Resolution(viewWidth, contentHeight * zoomFactor) } - } - private fun computeNormalResolution(): Resolution - { - if (viewHeight > viewWidth * contentAspect) - { - return Resolution(viewWidth, viewWidth * contentAspect) - } + private val normalResolution get() = + if(viewHeight > viewWidth * contentAspect) + Resolution(viewWidth, viewWidth * contentAspect) else - { - return Resolution(viewHeight / contentAspect, viewHeight) - } - } - + Resolution(viewHeight / contentAspect, viewHeight) } diff --git a/android/app/src/main/res/layout/activity_stream.xml b/android/app/src/main/res/layout/activity_stream.xml index 255ccde..9c944c3 100644 --- a/android/app/src/main/res/layout/activity_stream.xml +++ b/android/app/src/main/res/layout/activity_stream.xml @@ -1,16 +1,24 @@ - - + android:layout_height="match_parent" + android:layout_gravity="center"> + +