oxen-io
/
session-android
mirror of https://github.com/oxen-io/session-android.git
1
0
Fork 0
Code Issues 1 Projects Releases Wiki Activity

refactor: applying rotation and mirroring based on front / rear cameras that wraps nicely, only scale reworking needed

This commit is contained in:
jubb 2022-02-16 16:08:25 +11:00
parent a11a5da7c2
commit 0275edfcf9
8 changed files with 69 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
app/src/main/java/org/thoughtcrime/securesms/calls/WebRtcCallActivity.kt
View File

@ -44,8 +44,6 @@ import org.thoughtcrime.securesms.webrtc.CallViewModel.State.CALL_PRE_INIT
import org.thoughtcrime.securesms.webrtc.CallViewModel.State.CALL_RINGING
import org.thoughtcrime.securesms.webrtc.audio.SignalAudioManager.AudioDevice.EARPIECE
import org.thoughtcrime.securesms.webrtc.audio.SignalAudioManager.AudioDevice.SPEAKER_PHONE
import org.thoughtcrime.securesms.webrtc.data.quadrantRotation
import org.thoughtcrime.securesms.webrtc.video.RotationVideoProcessor
@AndroidEntryPoint
class WebRtcCallActivity : PassphraseRequiredActionBarActivity() {

25
app/src/main/java/org/thoughtcrime/securesms/webrtc/CallManager.kt
View File

@ -2,11 +2,8 @@ package org.thoughtcrime.securesms.webrtc
import android.content.Context
import android.telephony.TelephonyManager
import android.view.ViewGroup
import android.view.ViewGroup.LayoutParams.*
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.asSharedFlow
import kotlinx.serialization.Serializable
import kotlinx.serialization.json.*
import nl.komponents.kovenant.Promise
import nl.komponents.kovenant.functional.bind
@ -28,13 +25,14 @@ import org.thoughtcrime.securesms.webrtc.audio.SignalAudioManager.AudioDevice
import org.thoughtcrime.securesms.webrtc.locks.LockManager
import org.thoughtcrime.securesms.webrtc.video.CameraEventListener
import org.thoughtcrime.securesms.webrtc.video.CameraState
import org.thoughtcrime.securesms.webrtc.video.RemoteRotationVideoProxySink
import org.webrtc.*
import org.webrtc.PeerConnection.IceConnectionState
import org.webrtc.RendererCommon.ScalingType.SCALE_ASPECT_BALANCED
import org.webrtc.RendererCommon.ScalingType.SCALE_ASPECT_FILL
import org.webrtc.RendererCommon.ScalingType.SCALE_ASPECT_FIT
import java.nio.ByteBuffer
import java.util.*
import java.util.concurrent.Executors
class CallManager(context: Context, audioManager: AudioManagerCompat, private val storage: StorageProtocol): PeerConnection.Observer,
SignalAudioManager.EventListener, CameraEventListener, DataChannel.Observer {
@ -137,6 +135,7 @@ class CallManager(context: Context, audioManager: AudioManagerCompat, private va
private val outgoingIceDebouncer = Debouncer(200L)
var localRenderer: SurfaceViewRenderer? = null
var remoteRotationSink: RemoteRotationVideoProxySink? = null
var remoteRenderer: SurfaceViewRenderer? = null
private var peerConnectionFactory: PeerConnectionFactory? = null
@ -184,24 +183,18 @@ class CallManager(context: Context, audioManager: AudioManagerCompat, private va
eglBase = base
localRenderer = SurfaceViewRenderer(context).apply {
setEnableHardwareScaler(true)
setScalingType(SCALE_ASPECT_FIT)
}
remoteRenderer = SurfaceViewRenderer(context).apply {
setEnableHardwareScaler(true)
setScalingType(SCALE_ASPECT_FIT)
}
remoteRotationSink = RemoteRotationVideoProxySink()
localRenderer?.init(base.eglBaseContext, null)
localRenderer?.setMirror(localCameraState.activeDirection == CameraState.Direction.FRONT)
remoteRenderer?.init(base.eglBaseContext, object: RendererCommon.RendererEvents {
override fun onFirstFrameRendered() {
}
override fun onFrameResolutionChanged(p0: Int, p1: Int, p2: Int) {
Log.d("Loki", "remote rotation: $p2")
}
})
remoteRenderer?.init(base.eglBaseContext, null)
remoteRotationSink!!.setSink(remoteRenderer!!)
val encoderFactory = DefaultVideoEncoderFactory(base.eglBaseContext, true, true)
val decoderFactory = DefaultVideoDecoderFactory(base.eglBaseContext)
@ -299,7 +292,7 @@ class CallManager(context: Context, audioManager: AudioManagerCompat, private va
if (stream.videoTracks != null && stream.videoTracks.size == 1) {
val videoTrack = stream.videoTracks.first()
videoTrack.setEnabled(true)
videoTrack.addSink(remoteRenderer)
videoTrack.addSink(remoteRotationSink)
}
}
@ -359,6 +352,7 @@ class CallManager(context: Context, audioManager: AudioManagerCompat, private va
peerConnection = null
localRenderer?.release()
remoteRotationSink?.release()
remoteRenderer?.release()
eglBase?.release()
@ -584,6 +578,7 @@ class CallManager(context: Context, audioManager: AudioManagerCompat, private va
fun setDeviceRotation(newRotation: Int) {
peerConnection?.setDeviceRotation(newRotation)
remoteRotationSink?.rotation = newRotation
}
fun handleWiredHeadsetChanged(present: Boolean) {

13
app/src/main/java/org/thoughtcrime/securesms/webrtc/PeerConnectionWrapper.kt
View File

@ -6,7 +6,6 @@ import org.session.libsignal.utilities.SettableFuture
import org.thoughtcrime.securesms.webrtc.video.Camera
import org.thoughtcrime.securesms.webrtc.video.CameraEventListener
import org.thoughtcrime.securesms.webrtc.video.CameraState
import org.thoughtcrime.securesms.webrtc.video.RotationVideoProcessor
import org.thoughtcrime.securesms.webrtc.video.RotationVideoSink
import org.webrtc.*
import java.security.SecureRandom
@ -17,9 +16,9 @@ class PeerConnectionWrapper(context: Context,
factory: PeerConnectionFactory,
observer: PeerConnection.Observer,
localRenderer: VideoSink,
cameraEventListener: CameraEventListener,
private val cameraEventListener: CameraEventListener,
eglBase: EglBase,
relay: Boolean = false) {
relay: Boolean = false): CameraEventListener {
private val peerConnection: PeerConnection
private val audioTrack: AudioTrack
@ -66,7 +65,7 @@ class PeerConnectionWrapper(context: Context,
audioTrack.setEnabled(true)
mediaStream.addTrack(audioTrack)
camera = Camera(context, cameraEventListener)
camera = Camera(context, this)
if (camera.capturer != null) {
videoSource = factory.createVideoSource(false)
videoTrack = factory.createVideoTrack("ARDAMSv0", videoSource)
@ -76,6 +75,7 @@ class PeerConnectionWrapper(context: Context,
context,
rotationVideoSink
)
rotationVideoSink.mirrored = camera.activeDirection == CameraState.Direction.FRONT
rotationVideoSink.setSink(localRenderer)
videoTrack.setEnabled(false)
mediaStream.addTrack(videoTrack)
@ -297,4 +297,9 @@ class PeerConnectionWrapper(context: Context,
camera.flip()
}
override fun onCameraSwitchCompleted(newCameraState: CameraState) {
// mirror rotation offset
rotationVideoSink.mirrored = newCameraState.activeDirection == CameraState.Direction.FRONT
cameraEventListener.onCameraSwitchCompleted(newCameraState)
}
}

13
app/src/main/java/org/thoughtcrime/securesms/webrtc/data/CallUtils.kt
View File

@ -4,8 +4,13 @@ package org.thoughtcrime.securesms.webrtc.data
// chunks offset by 45 degrees
fun Int.quadrantRotation() = when (this % 360) {
in 315 until 360,
in 0 until 45 -> 90
in 45 until 135 -> 180
in 135 until 225 -> 270
else -> 0
in 0 until 45 -> 0
in 45 until 135 -> 90
in 135 until 225 -> 180
else -> 270
// in 315 until 360,
// in 0 until 45 -> 90
// in 45 until 135 -> 180
// in 135 until 225 -> 270
// else -> 0
}

33
app/src/main/java/org/thoughtcrime/securesms/webrtc/video/RemoteRotationVideoProxySink.kt Normal file
View File

@ -0,0 +1,33 @@
package org.thoughtcrime.securesms.webrtc.video
import org.session.libsignal.utilities.Log
import org.thoughtcrime.securesms.webrtc.data.quadrantRotation
import org.webrtc.VideoFrame
import org.webrtc.VideoSink
class RemoteRotationVideoProxySink: VideoSink {
private var targetSink: VideoSink? = null
var rotation: Int = 0
override fun onFrame(frame: VideoFrame?) {
val thisSink = targetSink ?: return
val thisFrame = frame ?: return
val quadrantRotation = rotation.quadrantRotation()
val newFrame = VideoFrame(thisFrame.buffer, (thisFrame.rotation - quadrantRotation) % 360, thisFrame.timestampNs)
thisSink.onFrame(newFrame)
}
fun setSink(videoSink: VideoSink) {
targetSink = videoSink
}
fun release() {
targetSink = null
}
}

35
app/src/main/java/org/thoughtcrime/securesms/webrtc/video/RotationVideoProcessor.kt
View File

@ -1,35 +0,0 @@
package org.thoughtcrime.securesms.webrtc.video
import kotlinx.coroutines.newSingleThreadContext
import org.webrtc.VideoFrame
import org.webrtc.VideoProcessor
import org.webrtc.VideoSink
class RotationVideoProcessor: VideoProcessor {
private var isCapturing: Boolean = true
private var sink: VideoSink? = null
var rotation: Int = 0
override fun onCapturerStarted(p0: Boolean) {
isCapturing = true
}
override fun onCapturerStopped() {
isCapturing = false
}
override fun onFrameCaptured(frame: VideoFrame?) {
val thisSink = sink ?: return
val thisFrame = frame ?: return
val newFrame = VideoFrame(thisFrame.buffer, rotation, thisFrame.timestampNs)
thisSink.onFrame(newFrame)
}
override fun setSink(newSink: VideoSink?) {
sink = newSink
}
}

8
app/src/main/java/org/thoughtcrime/securesms/webrtc/video/RotationVideoSink.kt
View File

@ -1,5 +1,6 @@
package org.thoughtcrime.securesms.webrtc.video
import org.session.libsignal.utilities.Log
import org.thoughtcrime.securesms.webrtc.data.quadrantRotation
import org.webrtc.CapturerObserver
import org.webrtc.VideoFrame
@ -11,6 +12,7 @@ import java.util.concurrent.atomic.AtomicBoolean
class RotationVideoSink: CapturerObserver, VideoProcessor {
var rotation: Int = 0
var mirrored = false
private val capturing = AtomicBoolean(false)
private var capturerObserver = SoftReference<CapturerObserver>(null)
@ -31,10 +33,8 @@ class RotationVideoSink: CapturerObserver, VideoProcessor {
val quadrantRotation = rotation.quadrantRotation()
val localRotation = 90
val newFrame = VideoFrame(videoFrame.buffer, quadrantRotation, videoFrame.timestampNs)
val localFrame = VideoFrame(videoFrame.buffer, localRotation, videoFrame.timestampNs)
val newFrame = VideoFrame(videoFrame.buffer, (videoFrame.rotation + quadrantRotation * if (mirrored && quadrantRotation in listOf(90,270)) -1 else 1) % 360, videoFrame.timestampNs)
val localFrame = VideoFrame(videoFrame.buffer, videoFrame.rotation * if (mirrored && quadrantRotation in listOf(90,270)) -1 else 1, videoFrame.timestampNs)
observer.onFrameCaptured(newFrame)
sink.get()?.onFrame(localFrame)

8
app/src/main/res/layout/activity_webrtc.xml
View File

@ -9,8 +9,8 @@
<FrameLayout
android:id="@+id/remote_parent"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_width="match_parent"
android:layout_height="match_parent"
app:layout_constraintStart_toStartOf="parent"
app:layout_constraintEnd_toEndOf="parent"
app:layout_constraintBottom_toBottomOf="parent"
@ -18,8 +18,8 @@
<FrameLayout
android:id="@+id/remote_renderer"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_width="match_parent"
android:layout_height="match_parent"
android:layout_gravity="center"/>
</FrameLayout>
<ImageView