From 0275edfcf950b3417b99f22e239bba619114e153 Mon Sep 17 00:00:00 2001 From: jubb Date: Wed, 16 Feb 2022 16:08:25 +1100 Subject: [PATCH] refactor: applying rotation and mirroring based on front / rear cameras that wraps nicely, only scale reworking needed --- .../securesms/calls/WebRtcCallActivity.kt | 2 -- .../securesms/webrtc/CallManager.kt | 25 ++++++------- .../securesms/webrtc/PeerConnectionWrapper.kt | 13 ++++--- .../securesms/webrtc/data/CallUtils.kt | 13 ++++--- .../video/RemoteRotationVideoProxySink.kt | 33 +++++++++++++++++ .../webrtc/video/RotationVideoProcessor.kt | 35 ------------------- .../webrtc/video/RotationVideoSink.kt | 8 ++--- app/src/main/res/layout/activity_webrtc.xml | 8 ++--- 8 files changed, 69 insertions(+), 68 deletions(-) create mode 100644 app/src/main/java/org/thoughtcrime/securesms/webrtc/video/RemoteRotationVideoProxySink.kt delete mode 100644 app/src/main/java/org/thoughtcrime/securesms/webrtc/video/RotationVideoProcessor.kt diff --git a/app/src/main/java/org/thoughtcrime/securesms/calls/WebRtcCallActivity.kt b/app/src/main/java/org/thoughtcrime/securesms/calls/WebRtcCallActivity.kt index 3836458c5..5736d5374 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/calls/WebRtcCallActivity.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/calls/WebRtcCallActivity.kt @@ -44,8 +44,6 @@ import org.thoughtcrime.securesms.webrtc.CallViewModel.State.CALL_PRE_INIT import org.thoughtcrime.securesms.webrtc.CallViewModel.State.CALL_RINGING import org.thoughtcrime.securesms.webrtc.audio.SignalAudioManager.AudioDevice.EARPIECE import org.thoughtcrime.securesms.webrtc.audio.SignalAudioManager.AudioDevice.SPEAKER_PHONE -import org.thoughtcrime.securesms.webrtc.data.quadrantRotation -import org.thoughtcrime.securesms.webrtc.video.RotationVideoProcessor @AndroidEntryPoint class WebRtcCallActivity : PassphraseRequiredActionBarActivity() { diff --git a/app/src/main/java/org/thoughtcrime/securesms/webrtc/CallManager.kt b/app/src/main/java/org/thoughtcrime/securesms/webrtc/CallManager.kt index 454d630e5..822c303ee 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/webrtc/CallManager.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/webrtc/CallManager.kt @@ -2,11 +2,8 @@ package org.thoughtcrime.securesms.webrtc import android.content.Context import android.telephony.TelephonyManager -import android.view.ViewGroup -import android.view.ViewGroup.LayoutParams.* import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.asSharedFlow -import kotlinx.serialization.Serializable import kotlinx.serialization.json.* import nl.komponents.kovenant.Promise import nl.komponents.kovenant.functional.bind @@ -28,13 +25,14 @@ import org.thoughtcrime.securesms.webrtc.audio.SignalAudioManager.AudioDevice import org.thoughtcrime.securesms.webrtc.locks.LockManager import org.thoughtcrime.securesms.webrtc.video.CameraEventListener import org.thoughtcrime.securesms.webrtc.video.CameraState +import org.thoughtcrime.securesms.webrtc.video.RemoteRotationVideoProxySink import org.webrtc.* import org.webrtc.PeerConnection.IceConnectionState +import org.webrtc.RendererCommon.ScalingType.SCALE_ASPECT_BALANCED import org.webrtc.RendererCommon.ScalingType.SCALE_ASPECT_FILL import org.webrtc.RendererCommon.ScalingType.SCALE_ASPECT_FIT import java.nio.ByteBuffer import java.util.* -import java.util.concurrent.Executors class CallManager(context: Context, audioManager: AudioManagerCompat, private val storage: StorageProtocol): PeerConnection.Observer, SignalAudioManager.EventListener, CameraEventListener, DataChannel.Observer { @@ -137,6 +135,7 @@ class CallManager(context: Context, audioManager: AudioManagerCompat, private va private val outgoingIceDebouncer = Debouncer(200L) var localRenderer: SurfaceViewRenderer? = null + var remoteRotationSink: RemoteRotationVideoProxySink? = null var remoteRenderer: SurfaceViewRenderer? = null private var peerConnectionFactory: PeerConnectionFactory? = null @@ -184,24 +183,18 @@ class CallManager(context: Context, audioManager: AudioManagerCompat, private va eglBase = base localRenderer = SurfaceViewRenderer(context).apply { setEnableHardwareScaler(true) - setScalingType(SCALE_ASPECT_FIT) } remoteRenderer = SurfaceViewRenderer(context).apply { setEnableHardwareScaler(true) - setScalingType(SCALE_ASPECT_FIT) } + remoteRotationSink = RemoteRotationVideoProxySink() + localRenderer?.init(base.eglBaseContext, null) localRenderer?.setMirror(localCameraState.activeDirection == CameraState.Direction.FRONT) - remoteRenderer?.init(base.eglBaseContext, object: RendererCommon.RendererEvents { - override fun onFirstFrameRendered() { - } - - override fun onFrameResolutionChanged(p0: Int, p1: Int, p2: Int) { - Log.d("Loki", "remote rotation: $p2") - } - }) + remoteRenderer?.init(base.eglBaseContext, null) + remoteRotationSink!!.setSink(remoteRenderer!!) val encoderFactory = DefaultVideoEncoderFactory(base.eglBaseContext, true, true) val decoderFactory = DefaultVideoDecoderFactory(base.eglBaseContext) @@ -299,7 +292,7 @@ class CallManager(context: Context, audioManager: AudioManagerCompat, private va if (stream.videoTracks != null && stream.videoTracks.size == 1) { val videoTrack = stream.videoTracks.first() videoTrack.setEnabled(true) - videoTrack.addSink(remoteRenderer) + videoTrack.addSink(remoteRotationSink) } } @@ -359,6 +352,7 @@ class CallManager(context: Context, audioManager: AudioManagerCompat, private va peerConnection = null localRenderer?.release() + remoteRotationSink?.release() remoteRenderer?.release() eglBase?.release() @@ -584,6 +578,7 @@ class CallManager(context: Context, audioManager: AudioManagerCompat, private va fun setDeviceRotation(newRotation: Int) { peerConnection?.setDeviceRotation(newRotation) + remoteRotationSink?.rotation = newRotation } fun handleWiredHeadsetChanged(present: Boolean) { diff --git a/app/src/main/java/org/thoughtcrime/securesms/webrtc/PeerConnectionWrapper.kt b/app/src/main/java/org/thoughtcrime/securesms/webrtc/PeerConnectionWrapper.kt index f237becc5..c13a7033c 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/webrtc/PeerConnectionWrapper.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/webrtc/PeerConnectionWrapper.kt @@ -6,7 +6,6 @@ import org.session.libsignal.utilities.SettableFuture import org.thoughtcrime.securesms.webrtc.video.Camera import org.thoughtcrime.securesms.webrtc.video.CameraEventListener import org.thoughtcrime.securesms.webrtc.video.CameraState -import org.thoughtcrime.securesms.webrtc.video.RotationVideoProcessor import org.thoughtcrime.securesms.webrtc.video.RotationVideoSink import org.webrtc.* import java.security.SecureRandom @@ -17,9 +16,9 @@ class PeerConnectionWrapper(context: Context, factory: PeerConnectionFactory, observer: PeerConnection.Observer, localRenderer: VideoSink, - cameraEventListener: CameraEventListener, + private val cameraEventListener: CameraEventListener, eglBase: EglBase, - relay: Boolean = false) { + relay: Boolean = false): CameraEventListener { private val peerConnection: PeerConnection private val audioTrack: AudioTrack @@ -66,7 +65,7 @@ class PeerConnectionWrapper(context: Context, audioTrack.setEnabled(true) mediaStream.addTrack(audioTrack) - camera = Camera(context, cameraEventListener) + camera = Camera(context, this) if (camera.capturer != null) { videoSource = factory.createVideoSource(false) videoTrack = factory.createVideoTrack("ARDAMSv0", videoSource) @@ -76,6 +75,7 @@ class PeerConnectionWrapper(context: Context, context, rotationVideoSink ) + rotationVideoSink.mirrored = camera.activeDirection == CameraState.Direction.FRONT rotationVideoSink.setSink(localRenderer) videoTrack.setEnabled(false) mediaStream.addTrack(videoTrack) @@ -297,4 +297,9 @@ class PeerConnectionWrapper(context: Context, camera.flip() } + override fun onCameraSwitchCompleted(newCameraState: CameraState) { + // mirror rotation offset + rotationVideoSink.mirrored = newCameraState.activeDirection == CameraState.Direction.FRONT + cameraEventListener.onCameraSwitchCompleted(newCameraState) + } } \ No newline at end of file diff --git a/app/src/main/java/org/thoughtcrime/securesms/webrtc/data/CallUtils.kt b/app/src/main/java/org/thoughtcrime/securesms/webrtc/data/CallUtils.kt index ce5342957..d2754a033 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/webrtc/data/CallUtils.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/webrtc/data/CallUtils.kt @@ -4,8 +4,13 @@ package org.thoughtcrime.securesms.webrtc.data // chunks offset by 45 degrees fun Int.quadrantRotation() = when (this % 360) { in 315 until 360, - in 0 until 45 -> 90 - in 45 until 135 -> 180 - in 135 until 225 -> 270 - else -> 0 + in 0 until 45 -> 0 + in 45 until 135 -> 90 + in 135 until 225 -> 180 + else -> 270 +// in 315 until 360, +// in 0 until 45 -> 90 +// in 45 until 135 -> 180 +// in 135 until 225 -> 270 +// else -> 0 } \ No newline at end of file diff --git a/app/src/main/java/org/thoughtcrime/securesms/webrtc/video/RemoteRotationVideoProxySink.kt b/app/src/main/java/org/thoughtcrime/securesms/webrtc/video/RemoteRotationVideoProxySink.kt new file mode 100644 index 000000000..fb3caf255 --- /dev/null +++ b/app/src/main/java/org/thoughtcrime/securesms/webrtc/video/RemoteRotationVideoProxySink.kt @@ -0,0 +1,33 @@ +package org.thoughtcrime.securesms.webrtc.video + +import org.session.libsignal.utilities.Log +import org.thoughtcrime.securesms.webrtc.data.quadrantRotation +import org.webrtc.VideoFrame +import org.webrtc.VideoSink + +class RemoteRotationVideoProxySink: VideoSink { + + private var targetSink: VideoSink? = null + + var rotation: Int = 0 + + override fun onFrame(frame: VideoFrame?) { + val thisSink = targetSink ?: return + val thisFrame = frame ?: return + + val quadrantRotation = rotation.quadrantRotation() + + val newFrame = VideoFrame(thisFrame.buffer, (thisFrame.rotation - quadrantRotation) % 360, thisFrame.timestampNs) + + thisSink.onFrame(newFrame) + } + + fun setSink(videoSink: VideoSink) { + targetSink = videoSink + } + + fun release() { + targetSink = null + } + +} \ No newline at end of file diff --git a/app/src/main/java/org/thoughtcrime/securesms/webrtc/video/RotationVideoProcessor.kt b/app/src/main/java/org/thoughtcrime/securesms/webrtc/video/RotationVideoProcessor.kt deleted file mode 100644 index 27fa078da..000000000 --- a/app/src/main/java/org/thoughtcrime/securesms/webrtc/video/RotationVideoProcessor.kt +++ /dev/null @@ -1,35 +0,0 @@ -package org.thoughtcrime.securesms.webrtc.video - -import kotlinx.coroutines.newSingleThreadContext -import org.webrtc.VideoFrame -import org.webrtc.VideoProcessor -import org.webrtc.VideoSink - -class RotationVideoProcessor: VideoProcessor { - - private var isCapturing: Boolean = true - private var sink: VideoSink? = null - - var rotation: Int = 0 - - override fun onCapturerStarted(p0: Boolean) { - isCapturing = true - } - - override fun onCapturerStopped() { - isCapturing = false - } - - override fun onFrameCaptured(frame: VideoFrame?) { - val thisSink = sink ?: return - val thisFrame = frame ?: return - - val newFrame = VideoFrame(thisFrame.buffer, rotation, thisFrame.timestampNs) - - thisSink.onFrame(newFrame) - } - - override fun setSink(newSink: VideoSink?) { - sink = newSink - } -} \ No newline at end of file diff --git a/app/src/main/java/org/thoughtcrime/securesms/webrtc/video/RotationVideoSink.kt b/app/src/main/java/org/thoughtcrime/securesms/webrtc/video/RotationVideoSink.kt index c0bdf5a10..ec43daf2e 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/webrtc/video/RotationVideoSink.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/webrtc/video/RotationVideoSink.kt @@ -1,5 +1,6 @@ package org.thoughtcrime.securesms.webrtc.video +import org.session.libsignal.utilities.Log import org.thoughtcrime.securesms.webrtc.data.quadrantRotation import org.webrtc.CapturerObserver import org.webrtc.VideoFrame @@ -11,6 +12,7 @@ import java.util.concurrent.atomic.AtomicBoolean class RotationVideoSink: CapturerObserver, VideoProcessor { var rotation: Int = 0 + var mirrored = false private val capturing = AtomicBoolean(false) private var capturerObserver = SoftReference(null) @@ -31,10 +33,8 @@ class RotationVideoSink: CapturerObserver, VideoProcessor { val quadrantRotation = rotation.quadrantRotation() - val localRotation = 90 - - val newFrame = VideoFrame(videoFrame.buffer, quadrantRotation, videoFrame.timestampNs) - val localFrame = VideoFrame(videoFrame.buffer, localRotation, videoFrame.timestampNs) + val newFrame = VideoFrame(videoFrame.buffer, (videoFrame.rotation + quadrantRotation * if (mirrored && quadrantRotation in listOf(90,270)) -1 else 1) % 360, videoFrame.timestampNs) + val localFrame = VideoFrame(videoFrame.buffer, videoFrame.rotation * if (mirrored && quadrantRotation in listOf(90,270)) -1 else 1, videoFrame.timestampNs) observer.onFrameCaptured(newFrame) sink.get()?.onFrame(localFrame) diff --git a/app/src/main/res/layout/activity_webrtc.xml b/app/src/main/res/layout/activity_webrtc.xml index 669c6794c..61d6f9d7b 100644 --- a/app/src/main/res/layout/activity_webrtc.xml +++ b/app/src/main/res/layout/activity_webrtc.xml @@ -9,8 +9,8 @@