From 9119ea2d5e14f7b714890f591addb3b3acab0e8e Mon Sep 17 00:00:00 2001 From: jubb Date: Fri, 11 Feb 2022 16:55:45 +1100 Subject: [PATCH] feat: adding base for rotation and picking random subset of turn servers --- app/src/main/AndroidManifest.xml | 1 - .../securesms/calls/WebRtcCallActivity.kt | 25 +++++++++++- .../securesms/service/WebRtcCallService.kt | 3 +- .../securesms/webrtc/CallManager.kt | 40 ++++++++++++++++--- .../securesms/webrtc/CallViewModel.kt | 3 ++ .../securesms/webrtc/PeerConnectionWrapper.kt | 10 ++++- 6 files changed, 71 insertions(+), 11 deletions(-) diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index 7ea486d7d..8a208db84 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -297,7 +297,6 @@ android:exported="true" android:theme="@style/Theme.Session.DayNight.NoActionBar" /> + Log.d("Loki", "Consuming view model state $state") when (state) { CALL_RINGING -> { if (wantsToAnswer) { @@ -308,6 +309,13 @@ class WebRtcCallActivity : PassphraseRequiredActionBarActivity() { } } + launch { + viewModel.cameraRotations.collect { (localRotation, remoteRotation) -> + val screenRotation = getDeviceRotation() + Log.d("Loki", "local rotation: $localRotation, remote rotation: $remoteRotation, screen rotation: $screenRotation") + } + } + launch { viewModel.remoteVideoEnabledState.collect { isEnabled -> binding.remoteRenderer.removeAllViews() @@ -323,6 +331,21 @@ class WebRtcCallActivity : PassphraseRequiredActionBarActivity() { } } + private fun getDeviceRotation(): Int { + val rotation = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { + display?.rotation + } else { + windowManager.defaultDisplay.rotation + } + return when (rotation) { + Surface.ROTATION_0 -> 0 + Surface.ROTATION_180 -> 180 + Surface.ROTATION_270 -> 270 + Surface.ROTATION_90 -> 90 + else -> throw NullPointerException("Unrecognized rotation $rotation") + } + } + private fun getUserDisplayName(publicKey: String): String { val contact = DatabaseComponent.get(this).sessionContactDatabase().getContactWithSessionID(publicKey) diff --git a/app/src/main/java/org/thoughtcrime/securesms/service/WebRtcCallService.kt b/app/src/main/java/org/thoughtcrime/securesms/service/WebRtcCallService.kt index a7f76e1c3..04cf2a823 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/service/WebRtcCallService.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/service/WebRtcCallService.kt @@ -85,7 +85,7 @@ class WebRtcCallService: Service(), CallManager.WebRtcListener { const val EXTRA_WANTS_TO_ANSWER = "wants_to_answer" const val INVALID_NOTIFICATION_ID = -1 - private const val TIMEOUT_SECONDS = 30L + private const val TIMEOUT_SECONDS = 60L fun cameraEnabled(context: Context, enabled: Boolean) = Intent(context, WebRtcCallService::class.java) .setAction(ACTION_SET_MUTE_VIDEO) @@ -485,7 +485,6 @@ class WebRtcCallService: Service(), CallManager.WebRtcListener { Log.e(TAG,e) terminate() } - // DatabaseComponent.get(this).smsDatabase().insertReceivedCall(recipient) } private fun handleDenyCall(intent: Intent) { diff --git a/app/src/main/java/org/thoughtcrime/securesms/webrtc/CallManager.kt b/app/src/main/java/org/thoughtcrime/securesms/webrtc/CallManager.kt index 45e30bf70..4afef2e93 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/webrtc/CallManager.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/webrtc/CallManager.kt @@ -28,6 +28,7 @@ import org.thoughtcrime.securesms.webrtc.video.CameraEventListener import org.thoughtcrime.securesms.webrtc.video.CameraState import org.webrtc.* import org.webrtc.PeerConnection.IceConnectionState +import org.webrtc.RendererCommon.ScalingType.SCALE_ASPECT_FIT import java.nio.ByteBuffer import java.util.* import java.util.concurrent.Executors @@ -99,6 +100,9 @@ class CallManager(context: Context, audioManager: AudioManagerCompat, private va val recipientEvents = _recipientEvents.asSharedFlow() private var localCameraState: CameraState = CameraState.UNKNOWN + private val _cameraRotations = MutableStateFlow(0 to 0) + val cameraRotations = _cameraRotations.asSharedFlow() + private val _audioDeviceEvents = MutableStateFlow(AudioDeviceUpdate(AudioDevice.NONE, setOf())) val audioDeviceEvents = _audioDeviceEvents.asSharedFlow() @@ -161,6 +165,7 @@ class CallManager(context: Context, audioManager: AudioManagerCompat, private va } fun postViewModelState(newState: CallViewModel.State) { + Log.d("Loki", "Posting view model state $newState") _callStateEvents.value = newState } @@ -177,12 +182,35 @@ class CallManager(context: Context, audioManager: AudioManagerCompat, private va Util.runOnMainSync { val base = EglBase.create() eglBase = base - localRenderer = SurfaceViewRenderer(context) - remoteRenderer = SurfaceViewRenderer(context) + _cameraRotations.value = 0 to 0 + localRenderer = SurfaceViewRenderer(context).apply { + setEnableHardwareScaler(true) + } - localRenderer?.init(base.eglBaseContext, null) - localRenderer?.setMirror(true) - remoteRenderer?.init(base.eglBaseContext, null) + remoteRenderer = SurfaceViewRenderer(context).apply { + setEnableHardwareScaler(true) + setScalingType(SCALE_ASPECT_FIT, SCALE_ASPECT_FIT) + } + + localRenderer?.init(base.eglBaseContext, object : RendererCommon.RendererEvents { + override fun onFirstFrameRendered() { + localCameraState + } + + override fun onFrameResolutionChanged(p0: Int, p1: Int, rotation: Int) { + _cameraRotations.value = _cameraRotations.value.copy(first = rotation) + } + }) + localRenderer?.setMirror(localCameraState.activeDirection == CameraState.Direction.FRONT) + remoteRenderer?.init(base.eglBaseContext, object : RendererCommon.RendererEvents { + override fun onFirstFrameRendered() { + localCameraState + } + + override fun onFrameResolutionChanged(p0: Int, p1: Int, rotation: Int) { + _cameraRotations.value = _cameraRotations.value.copy(second = rotation) + } + }) val encoderFactory = DefaultVideoEncoderFactory(base.eglBaseContext, true, true) val decoderFactory = DefaultVideoDecoderFactory(base.eglBaseContext) @@ -216,6 +244,7 @@ class CallManager(context: Context, audioManager: AudioManagerCompat, private va } override fun onIceConnectionChange(newState: IceConnectionState) { + Log.d("Loki", "New ice connection state = $newState") iceState = newState peerConnectionObservers.forEach { listener -> listener.onIceConnectionChange(newState) } if (newState == IceConnectionState.CONNECTED) { @@ -558,6 +587,7 @@ class CallManager(context: Context, audioManager: AudioManagerCompat, private va peerConnection?.let { connection -> connection.flipCamera() localCameraState = connection.getCameraState() + localRenderer?.setMirror(localCameraState.activeDirection == CameraState.Direction.FRONT) } } diff --git a/app/src/main/java/org/thoughtcrime/securesms/webrtc/CallViewModel.kt b/app/src/main/java/org/thoughtcrime/securesms/webrtc/CallViewModel.kt index 20d9eba67..a13892f20 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/webrtc/CallViewModel.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/webrtc/CallViewModel.kt @@ -66,6 +66,9 @@ class CallViewModel @Inject constructor(private val callManager: CallManager): V val remoteVideoEnabledState get() = callManager.remoteVideoEvents.map { it.isEnabled } + val cameraRotations + get() = callManager.cameraRotations + val currentCallState get() = callManager.currentCallState diff --git a/app/src/main/java/org/thoughtcrime/securesms/webrtc/PeerConnectionWrapper.kt b/app/src/main/java/org/thoughtcrime/securesms/webrtc/PeerConnectionWrapper.kt index e12a358c2..6e332db53 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/webrtc/PeerConnectionWrapper.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/webrtc/PeerConnectionWrapper.kt @@ -6,7 +6,9 @@ import org.thoughtcrime.securesms.webrtc.video.Camera import org.thoughtcrime.securesms.webrtc.video.CameraEventListener import org.thoughtcrime.securesms.webrtc.video.CameraState import org.webrtc.* +import java.security.SecureRandom import java.util.concurrent.ExecutionException +import kotlin.random.asKotlinRandom class PeerConnectionWrapper(context: Context, factory: PeerConnectionFactory, @@ -27,8 +29,12 @@ class PeerConnectionWrapper(context: Context, get() = peerConnection.localDescription != null && peerConnection.remoteDescription != null init { - val iceServers = listOf("freyr","fenrir","frigg","angus","hereford","holstein","brahman").map { sub -> - PeerConnection.IceServer.builder("turn:$sub.getsession.org").setUsername("session202111").setPassword("053c268164bc7bd7").createIceServer() + val random = SecureRandom().asKotlinRandom() + val iceServers = listOf("freyr","fenrir","frigg","angus","hereford","holstein", "brahman").shuffled(random).take(2).map { sub -> + PeerConnection.IceServer.builder("turn:$sub.getsession.org") + .setUsername("session202111") + .setPassword("053c268164bc7bd7") + .createIceServer() } val constraints = MediaConstraints().apply {