refactor: applying rotation and mirroring based on front / rear cameras that wraps nicely, only scale reworking needed
This commit is contained in:
parent
a11a5da7c2
commit
0275edfcf9
|
@ -44,8 +44,6 @@ import org.thoughtcrime.securesms.webrtc.CallViewModel.State.CALL_PRE_INIT
|
|||
import org.thoughtcrime.securesms.webrtc.CallViewModel.State.CALL_RINGING
|
||||
import org.thoughtcrime.securesms.webrtc.audio.SignalAudioManager.AudioDevice.EARPIECE
|
||||
import org.thoughtcrime.securesms.webrtc.audio.SignalAudioManager.AudioDevice.SPEAKER_PHONE
|
||||
import org.thoughtcrime.securesms.webrtc.data.quadrantRotation
|
||||
import org.thoughtcrime.securesms.webrtc.video.RotationVideoProcessor
|
||||
|
||||
@AndroidEntryPoint
|
||||
class WebRtcCallActivity : PassphraseRequiredActionBarActivity() {
|
||||
|
|
|
@ -2,11 +2,8 @@ package org.thoughtcrime.securesms.webrtc
|
|||
|
||||
import android.content.Context
|
||||
import android.telephony.TelephonyManager
|
||||
import android.view.ViewGroup
|
||||
import android.view.ViewGroup.LayoutParams.*
|
||||
import kotlinx.coroutines.flow.MutableStateFlow
|
||||
import kotlinx.coroutines.flow.asSharedFlow
|
||||
import kotlinx.serialization.Serializable
|
||||
import kotlinx.serialization.json.*
|
||||
import nl.komponents.kovenant.Promise
|
||||
import nl.komponents.kovenant.functional.bind
|
||||
|
@ -28,13 +25,14 @@ import org.thoughtcrime.securesms.webrtc.audio.SignalAudioManager.AudioDevice
|
|||
import org.thoughtcrime.securesms.webrtc.locks.LockManager
|
||||
import org.thoughtcrime.securesms.webrtc.video.CameraEventListener
|
||||
import org.thoughtcrime.securesms.webrtc.video.CameraState
|
||||
import org.thoughtcrime.securesms.webrtc.video.RemoteRotationVideoProxySink
|
||||
import org.webrtc.*
|
||||
import org.webrtc.PeerConnection.IceConnectionState
|
||||
import org.webrtc.RendererCommon.ScalingType.SCALE_ASPECT_BALANCED
|
||||
import org.webrtc.RendererCommon.ScalingType.SCALE_ASPECT_FILL
|
||||
import org.webrtc.RendererCommon.ScalingType.SCALE_ASPECT_FIT
|
||||
import java.nio.ByteBuffer
|
||||
import java.util.*
|
||||
import java.util.concurrent.Executors
|
||||
|
||||
class CallManager(context: Context, audioManager: AudioManagerCompat, private val storage: StorageProtocol): PeerConnection.Observer,
|
||||
SignalAudioManager.EventListener, CameraEventListener, DataChannel.Observer {
|
||||
|
@ -137,6 +135,7 @@ class CallManager(context: Context, audioManager: AudioManagerCompat, private va
|
|||
private val outgoingIceDebouncer = Debouncer(200L)
|
||||
|
||||
var localRenderer: SurfaceViewRenderer? = null
|
||||
var remoteRotationSink: RemoteRotationVideoProxySink? = null
|
||||
var remoteRenderer: SurfaceViewRenderer? = null
|
||||
private var peerConnectionFactory: PeerConnectionFactory? = null
|
||||
|
||||
|
@ -184,24 +183,18 @@ class CallManager(context: Context, audioManager: AudioManagerCompat, private va
|
|||
eglBase = base
|
||||
localRenderer = SurfaceViewRenderer(context).apply {
|
||||
setEnableHardwareScaler(true)
|
||||
setScalingType(SCALE_ASPECT_FIT)
|
||||
}
|
||||
|
||||
remoteRenderer = SurfaceViewRenderer(context).apply {
|
||||
setEnableHardwareScaler(true)
|
||||
setScalingType(SCALE_ASPECT_FIT)
|
||||
}
|
||||
remoteRotationSink = RemoteRotationVideoProxySink()
|
||||
|
||||
|
||||
localRenderer?.init(base.eglBaseContext, null)
|
||||
localRenderer?.setMirror(localCameraState.activeDirection == CameraState.Direction.FRONT)
|
||||
remoteRenderer?.init(base.eglBaseContext, object: RendererCommon.RendererEvents {
|
||||
override fun onFirstFrameRendered() {
|
||||
}
|
||||
|
||||
override fun onFrameResolutionChanged(p0: Int, p1: Int, p2: Int) {
|
||||
Log.d("Loki", "remote rotation: $p2")
|
||||
}
|
||||
})
|
||||
remoteRenderer?.init(base.eglBaseContext, null)
|
||||
remoteRotationSink!!.setSink(remoteRenderer!!)
|
||||
|
||||
val encoderFactory = DefaultVideoEncoderFactory(base.eglBaseContext, true, true)
|
||||
val decoderFactory = DefaultVideoDecoderFactory(base.eglBaseContext)
|
||||
|
@ -299,7 +292,7 @@ class CallManager(context: Context, audioManager: AudioManagerCompat, private va
|
|||
if (stream.videoTracks != null && stream.videoTracks.size == 1) {
|
||||
val videoTrack = stream.videoTracks.first()
|
||||
videoTrack.setEnabled(true)
|
||||
videoTrack.addSink(remoteRenderer)
|
||||
videoTrack.addSink(remoteRotationSink)
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -359,6 +352,7 @@ class CallManager(context: Context, audioManager: AudioManagerCompat, private va
|
|||
peerConnection = null
|
||||
|
||||
localRenderer?.release()
|
||||
remoteRotationSink?.release()
|
||||
remoteRenderer?.release()
|
||||
eglBase?.release()
|
||||
|
||||
|
@ -584,6 +578,7 @@ class CallManager(context: Context, audioManager: AudioManagerCompat, private va
|
|||
|
||||
fun setDeviceRotation(newRotation: Int) {
|
||||
peerConnection?.setDeviceRotation(newRotation)
|
||||
remoteRotationSink?.rotation = newRotation
|
||||
}
|
||||
|
||||
fun handleWiredHeadsetChanged(present: Boolean) {
|
||||
|
|
|
@ -6,7 +6,6 @@ import org.session.libsignal.utilities.SettableFuture
|
|||
import org.thoughtcrime.securesms.webrtc.video.Camera
|
||||
import org.thoughtcrime.securesms.webrtc.video.CameraEventListener
|
||||
import org.thoughtcrime.securesms.webrtc.video.CameraState
|
||||
import org.thoughtcrime.securesms.webrtc.video.RotationVideoProcessor
|
||||
import org.thoughtcrime.securesms.webrtc.video.RotationVideoSink
|
||||
import org.webrtc.*
|
||||
import java.security.SecureRandom
|
||||
|
@ -17,9 +16,9 @@ class PeerConnectionWrapper(context: Context,
|
|||
factory: PeerConnectionFactory,
|
||||
observer: PeerConnection.Observer,
|
||||
localRenderer: VideoSink,
|
||||
cameraEventListener: CameraEventListener,
|
||||
private val cameraEventListener: CameraEventListener,
|
||||
eglBase: EglBase,
|
||||
relay: Boolean = false) {
|
||||
relay: Boolean = false): CameraEventListener {
|
||||
|
||||
private val peerConnection: PeerConnection
|
||||
private val audioTrack: AudioTrack
|
||||
|
@ -66,7 +65,7 @@ class PeerConnectionWrapper(context: Context,
|
|||
audioTrack.setEnabled(true)
|
||||
mediaStream.addTrack(audioTrack)
|
||||
|
||||
camera = Camera(context, cameraEventListener)
|
||||
camera = Camera(context, this)
|
||||
if (camera.capturer != null) {
|
||||
videoSource = factory.createVideoSource(false)
|
||||
videoTrack = factory.createVideoTrack("ARDAMSv0", videoSource)
|
||||
|
@ -76,6 +75,7 @@ class PeerConnectionWrapper(context: Context,
|
|||
context,
|
||||
rotationVideoSink
|
||||
)
|
||||
rotationVideoSink.mirrored = camera.activeDirection == CameraState.Direction.FRONT
|
||||
rotationVideoSink.setSink(localRenderer)
|
||||
videoTrack.setEnabled(false)
|
||||
mediaStream.addTrack(videoTrack)
|
||||
|
@ -297,4 +297,9 @@ class PeerConnectionWrapper(context: Context,
|
|||
camera.flip()
|
||||
}
|
||||
|
||||
override fun onCameraSwitchCompleted(newCameraState: CameraState) {
|
||||
// mirror rotation offset
|
||||
rotationVideoSink.mirrored = newCameraState.activeDirection == CameraState.Direction.FRONT
|
||||
cameraEventListener.onCameraSwitchCompleted(newCameraState)
|
||||
}
|
||||
}
|
|
@ -4,8 +4,13 @@ package org.thoughtcrime.securesms.webrtc.data
|
|||
// chunks offset by 45 degrees
|
||||
fun Int.quadrantRotation() = when (this % 360) {
|
||||
in 315 until 360,
|
||||
in 0 until 45 -> 90
|
||||
in 45 until 135 -> 180
|
||||
in 135 until 225 -> 270
|
||||
else -> 0
|
||||
in 0 until 45 -> 0
|
||||
in 45 until 135 -> 90
|
||||
in 135 until 225 -> 180
|
||||
else -> 270
|
||||
// in 315 until 360,
|
||||
// in 0 until 45 -> 90
|
||||
// in 45 until 135 -> 180
|
||||
// in 135 until 225 -> 270
|
||||
// else -> 0
|
||||
}
|
|
@ -0,0 +1,33 @@
|
|||
package org.thoughtcrime.securesms.webrtc.video
|
||||
|
||||
import org.session.libsignal.utilities.Log
|
||||
import org.thoughtcrime.securesms.webrtc.data.quadrantRotation
|
||||
import org.webrtc.VideoFrame
|
||||
import org.webrtc.VideoSink
|
||||
|
||||
class RemoteRotationVideoProxySink: VideoSink {
|
||||
|
||||
private var targetSink: VideoSink? = null
|
||||
|
||||
var rotation: Int = 0
|
||||
|
||||
override fun onFrame(frame: VideoFrame?) {
|
||||
val thisSink = targetSink ?: return
|
||||
val thisFrame = frame ?: return
|
||||
|
||||
val quadrantRotation = rotation.quadrantRotation()
|
||||
|
||||
val newFrame = VideoFrame(thisFrame.buffer, (thisFrame.rotation - quadrantRotation) % 360, thisFrame.timestampNs)
|
||||
|
||||
thisSink.onFrame(newFrame)
|
||||
}
|
||||
|
||||
fun setSink(videoSink: VideoSink) {
|
||||
targetSink = videoSink
|
||||
}
|
||||
|
||||
fun release() {
|
||||
targetSink = null
|
||||
}
|
||||
|
||||
}
|
|
@ -1,35 +0,0 @@
|
|||
package org.thoughtcrime.securesms.webrtc.video
|
||||
|
||||
import kotlinx.coroutines.newSingleThreadContext
|
||||
import org.webrtc.VideoFrame
|
||||
import org.webrtc.VideoProcessor
|
||||
import org.webrtc.VideoSink
|
||||
|
||||
class RotationVideoProcessor: VideoProcessor {
|
||||
|
||||
private var isCapturing: Boolean = true
|
||||
private var sink: VideoSink? = null
|
||||
|
||||
var rotation: Int = 0
|
||||
|
||||
override fun onCapturerStarted(p0: Boolean) {
|
||||
isCapturing = true
|
||||
}
|
||||
|
||||
override fun onCapturerStopped() {
|
||||
isCapturing = false
|
||||
}
|
||||
|
||||
override fun onFrameCaptured(frame: VideoFrame?) {
|
||||
val thisSink = sink ?: return
|
||||
val thisFrame = frame ?: return
|
||||
|
||||
val newFrame = VideoFrame(thisFrame.buffer, rotation, thisFrame.timestampNs)
|
||||
|
||||
thisSink.onFrame(newFrame)
|
||||
}
|
||||
|
||||
override fun setSink(newSink: VideoSink?) {
|
||||
sink = newSink
|
||||
}
|
||||
}
|
|
@ -1,5 +1,6 @@
|
|||
package org.thoughtcrime.securesms.webrtc.video
|
||||
|
||||
import org.session.libsignal.utilities.Log
|
||||
import org.thoughtcrime.securesms.webrtc.data.quadrantRotation
|
||||
import org.webrtc.CapturerObserver
|
||||
import org.webrtc.VideoFrame
|
||||
|
@ -11,6 +12,7 @@ import java.util.concurrent.atomic.AtomicBoolean
|
|||
class RotationVideoSink: CapturerObserver, VideoProcessor {
|
||||
|
||||
var rotation: Int = 0
|
||||
var mirrored = false
|
||||
|
||||
private val capturing = AtomicBoolean(false)
|
||||
private var capturerObserver = SoftReference<CapturerObserver>(null)
|
||||
|
@ -31,10 +33,8 @@ class RotationVideoSink: CapturerObserver, VideoProcessor {
|
|||
|
||||
val quadrantRotation = rotation.quadrantRotation()
|
||||
|
||||
val localRotation = 90
|
||||
|
||||
val newFrame = VideoFrame(videoFrame.buffer, quadrantRotation, videoFrame.timestampNs)
|
||||
val localFrame = VideoFrame(videoFrame.buffer, localRotation, videoFrame.timestampNs)
|
||||
val newFrame = VideoFrame(videoFrame.buffer, (videoFrame.rotation + quadrantRotation * if (mirrored && quadrantRotation in listOf(90,270)) -1 else 1) % 360, videoFrame.timestampNs)
|
||||
val localFrame = VideoFrame(videoFrame.buffer, videoFrame.rotation * if (mirrored && quadrantRotation in listOf(90,270)) -1 else 1, videoFrame.timestampNs)
|
||||
|
||||
observer.onFrameCaptured(newFrame)
|
||||
sink.get()?.onFrame(localFrame)
|
||||
|
|
|
@ -9,8 +9,8 @@
|
|||
|
||||
<FrameLayout
|
||||
android:id="@+id/remote_parent"
|
||||
android:layout_width="wrap_content"
|
||||
android:layout_height="wrap_content"
|
||||
android:layout_width="match_parent"
|
||||
android:layout_height="match_parent"
|
||||
app:layout_constraintStart_toStartOf="parent"
|
||||
app:layout_constraintEnd_toEndOf="parent"
|
||||
app:layout_constraintBottom_toBottomOf="parent"
|
||||
|
@ -18,8 +18,8 @@
|
|||
|
||||
<FrameLayout
|
||||
android:id="@+id/remote_renderer"
|
||||
android:layout_width="wrap_content"
|
||||
android:layout_height="wrap_content"
|
||||
android:layout_width="match_parent"
|
||||
android:layout_height="match_parent"
|
||||
android:layout_gravity="center"/>
|
||||
</FrameLayout>
|
||||
<ImageView
|
||||
|
|
Loading…
Reference in New Issue