feat: plugging CallManager.kt into view model and service, fixing up dependencies

This commit is contained in:
jubb 2021-10-29 16:41:01 +11:00
parent 71bb04cb34
commit 1af9b8ba46
14 changed files with 288 additions and 188 deletions

View File

@ -37,7 +37,6 @@ dependencies {
implementation 'androidx.gridlayout:gridlayout:1.0.0'
implementation 'androidx.exifinterface:exifinterface:1.2.0'
implementation 'androidx.constraintlayout:constraintlayout:2.0.1'
implementation 'androidx.lifecycle:lifecycle-extensions:2.4.0'
implementation 'androidx.lifecycle:lifecycle-common-java8:2.4.0'
implementation 'androidx.lifecycle:lifecycle-viewmodel-ktx:2.4.0'
implementation 'androidx.lifecycle:lifecycle-runtime-ktx:2.4.0'

View File

@ -5,15 +5,21 @@ import android.content.BroadcastReceiver
import android.content.Context
import android.content.Intent
import android.content.IntentFilter
import android.media.AudioManager
import android.os.Bundle
import android.view.MenuItem
import android.view.Window
import android.view.WindowManager
import androidx.activity.viewModels
import androidx.core.view.isVisible
import androidx.lifecycle.Lifecycle
import androidx.lifecycle.lifecycleScope
import androidx.lifecycle.repeatOnLifecycle
import dagger.hilt.android.AndroidEntryPoint
import kotlinx.android.synthetic.main.activity_webrtc_tests.*
import kotlinx.coroutines.delay
import kotlinx.coroutines.isActive
import kotlinx.coroutines.launch
import network.loki.messenger.R
import org.session.libsession.messaging.messages.control.CallMessage
import org.session.libsession.messaging.sending_receiving.MessageSender
@ -29,12 +35,9 @@ import org.webrtc.*
import java.util.*
@AndroidEntryPoint
class WebRtcTestsActivity: PassphraseRequiredActionBarActivity(), PeerConnection.Observer,
SdpObserver, RTCStatsCollectorCallback {
class WebRtcTestsActivity: PassphraseRequiredActionBarActivity() {
companion object {
const val HD_VIDEO_WIDTH = 900
const val HD_VIDEO_HEIGHT = 1600
const val CALL_ID = "call_id_session"
private const val LOCAL_TRACK_ID = "local_track"
private const val LOCAL_STREAM_ID = "local_track"
@ -45,39 +48,13 @@ class WebRtcTestsActivity: PassphraseRequiredActionBarActivity(), PeerConnection
const val EXTRA_SDP = "WebRtcTestsActivity_EXTRA_SDP"
const val EXTRA_ADDRESS = "WebRtcTestsActivity_EXTRA_ADDRESS"
const val EXTRA_CALL_ID = "WebRtcTestsActivity_EXTRA_CALL_ID"
}
private val viewModel by viewModels<CallViewModel>()
private val surfaceHelper by lazy { SurfaceTextureHelper.create(Thread.currentThread().name, viewModel.eglBaseContext) }
private val audioSource by lazy { connectionFactory.createAudioSource(MediaConstraints()) }
private val videoCapturer by lazy { createCameraCapturer(Camera2Enumerator(this)) }
private val acceptedCallMessageHashes = mutableSetOf<Int>()
private val candidates: MutableList<IceCandidate> = mutableListOf()
private val iceDebouncer = Debouncer(2_000)
private var localCandidateType: String? = null
set(value) {
field = value
if (value != null) {
// show it
local_candidate_info.isVisible = true
local_candidate_info.text = "local: $value"
}
}
private var remoteCandidateType: String? = null
set(value) {
field = value
if (value != null) {
remote_candidate_info.isVisible = true
remote_candidate_info.text = "remote: $value"
}
// update text
}
private lateinit var callAddress: Address
private lateinit var callId: UUID
@ -96,16 +73,27 @@ class WebRtcTestsActivity: PassphraseRequiredActionBarActivity(), PeerConnection
override fun onCreate(savedInstanceState: Bundle?, ready: Boolean) {
super.onCreate(savedInstanceState, ready)
window.addFlags(WindowManager.LayoutParams.FLAG_SHOW_WHEN_LOCKED)
window.addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON)
requestWindowFeature(Window.FEATURE_NO_TITLE)
setContentView(R.layout.activity_webrtc_tests)
volumeControlStream = AudioManager.STREAM_VOICE_CALL
initializeResources()
//TODO: better handling of permissions
Permissions.with(this)
.request(Manifest.permission.CAMERA, Manifest.permission.RECORD_AUDIO)
.request(Manifest.permission.RECORD_AUDIO)
.onAllGranted {
setupStreams()
}
.execute()
lifecycleScope.launch {
repeatOnLifecycle(Lifecycle.State.STARTED) {
viewModel
}
}
local_renderer.run {
setEnableHardwareScaler(true)
init(eglBase.eglBaseContext, null)
@ -180,24 +168,8 @@ class WebRtcTestsActivity: PassphraseRequiredActionBarActivity(), PeerConnection
}
}
override fun onStatsDelivered(statsReport: RTCStatsReport?) {
statsReport?.let { report ->
val usedConnection = report.statsMap.filter { (_,v) -> v.type == "candidate-pair" && v.members["writable"] == true }.asIterable().firstOrNull()?.value ?: return@let
private fun initializeResources() {
usedConnection.members["remoteCandidateId"]?.let { candidate ->
runOnUiThread {
remoteCandidateType = report.statsMap[candidate]?.members?.get("candidateType") as? String
}
}
usedConnection.members["localCandidateId"]?.let { candidate ->
runOnUiThread {
localCandidateType = report.statsMap[candidate]?.members?.get("candidateType") as? String
}
}
Log.d("Loki-RTC", "report is: $report")
}
}
private fun endCall() {
@ -237,36 +209,6 @@ class WebRtcTestsActivity: PassphraseRequiredActionBarActivity(), PeerConnection
peerConnection.addStream(stream)
}
private fun createCameraCapturer(enumerator: CameraEnumerator): CameraVideoCapturer? {
val deviceNames = enumerator.deviceNames
// First, try to find front facing camera
Log.d("Loki-RTC-vid", "Looking for front facing cameras.")
for (deviceName in deviceNames) {
if (enumerator.isFrontFacing(deviceName)) {
Log.d("Loki-RTC-vid", "Creating front facing camera capturer.")
val videoCapturer = enumerator.createCapturer(deviceName, null)
if (videoCapturer != null) {
return videoCapturer
}
}
}
// Front facing camera not found, try something else
Log.d("Loki-RTC-vid", "Looking for other cameras.")
for (deviceName in deviceNames) {
if (!enumerator.isFrontFacing(deviceName)) {
Log.d("Loki-RTC-vid", "Creating other camera capturer.")
val videoCapturer = enumerator.createCapturer(deviceName, null)
if (videoCapturer != null) {
return videoCapturer
}
}
}
return null
}
override fun onSignalingChange(p0: PeerConnection.SignalingState?) {
Log.d("Loki-RTC", "onSignalingChange: $p0")
}
@ -375,11 +317,4 @@ class WebRtcTestsActivity: PassphraseRequiredActionBarActivity(), PeerConnection
Log.d("Loki-RTC", "onSetFailure: $p0")
}
private fun CallMessage.iceCandidates(): List<IceCandidate> {
val candidateSize = sdpMids.size
return (0 until candidateSize).map { i ->
IceCandidate(sdpMids[i], sdpMLineIndexes[i], sdps[i])
}
}
}

View File

@ -16,6 +16,6 @@ interface CallComponent {
fun get(context: Context) = ApplicationContext.getInstance(context).callComponent
}
fun callManagerCompat(): AudioManagerCompat
fun audioManagerCompat(): AudioManagerCompat
}

View File

@ -1,20 +1,23 @@
package org.thoughtcrime.securesms.dependencies
import android.content.Context
import dagger.Binds
import dagger.Module
import dagger.Provides
import dagger.hilt.InstallIn
import dagger.hilt.android.qualifiers.ApplicationContext
import dagger.hilt.components.SingletonComponent
import org.session.libsession.database.CallDataProvider
import org.thoughtcrime.securesms.database.Storage
import org.thoughtcrime.securesms.webrtc.CallManager
import org.thoughtcrime.securesms.webrtc.audio.AudioManagerCompat
import org.thoughtcrime.securesms.webrtc.audio.SignalAudioManager
import org.thoughtcrime.securesms.webrtc.data.SessionCallDataProvider
import javax.inject.Singleton
@Module
@InstallIn(SingletonComponent::class)
object CallModule {
abstract class CallModule {
@Provides
@Singleton
@ -23,6 +26,10 @@ object CallModule {
@Provides
@Singleton
fun provideCallManager(@ApplicationContext context: Context, storage: Storage) =
CallManager(context, storage)
CallManager(context)
@Binds
@Singleton
abstract fun bindCallDataProvider(sessionCallDataProvider: SessionCallDataProvider): CallDataProvider
}

View File

@ -122,10 +122,6 @@ object DatabaseModule {
@Singleton
fun provideStorage(@ApplicationContext context: Context, openHelper: SQLCipherOpenHelper) = Storage(context,openHelper)
// @Provides
// @Singleton
// fun provideCallDataProvider(storage: Storage) = SessionCallDataProvider(storage)
@Provides
@Singleton
fun provideAttachmentProvider(@ApplicationContext context: Context, openHelper: SQLCipherOpenHelper): MessageDataProvider = DatabaseAttachmentProvider(context, openHelper)

View File

@ -18,16 +18,9 @@ import kotlin.properties.Delegates
import kotlin.properties.Delegates.observable
@AndroidEntryPoint
class WebRtcCallService: Service(), SignalAudioManager.EventListener {
class WebRtcCallService: Service() {
@Inject lateinit var callManager: CallManager
val signalAudioManager: SignalAudioManager by lazy {
SignalAudioManager(this, this, CallComponent.get(this).callManagerCompat())
}
private enum class CallState {
STATE_IDLE, STATE_DIALING, STATE_ANSWERING, STATE_REMOTE_RINGING, STATE_LOCAL_RINGING, STATE_CONNECTED
}
companion object {
private const val ACTION_UPDATE = "UPDATE"
@ -81,10 +74,6 @@ class WebRtcCallService: Service(), SignalAudioManager.EventListener {
}
}
private var state: CallState by observable(CallState.STATE_IDLE) { _, previousValue, newValue ->
}
override fun onBind(intent: Intent?): IBinder? = null
override fun onCreate() {
@ -103,8 +92,4 @@ class WebRtcCallService: Service(), SignalAudioManager.EventListener {
// unregister network receiver
// unregister power button
}
override fun onAudioDeviceChanged(activeDevice: SignalAudioManager.AudioDevice, devices: Set<SignalAudioManager.AudioDevice>) {
TODO("Not yet implemented")
}
}

View File

@ -0,0 +1,7 @@
package org.thoughtcrime.securesms.webrtc
import org.session.libsignal.protos.SignalServiceProtos
interface CallDataListener {
fun newCallMessage(callMessage: SignalServiceProtos.CallMessage)
}

View File

@ -2,44 +2,72 @@ package org.thoughtcrime.securesms.webrtc
import android.content.Context
import com.android.mms.transaction.MessageSender
import kotlinx.coroutines.flow.MutableStateFlow
import org.session.libsession.messaging.messages.control.CallMessage
import org.session.libsignal.protos.SignalServiceProtos
import org.session.libsignal.utilities.Log
import org.thoughtcrime.securesms.database.Storage
import org.thoughtcrime.securesms.dependencies.CallComponent
import org.thoughtcrime.securesms.service.WebRtcCallService
import org.thoughtcrime.securesms.webrtc.audio.SignalAudioManager
import org.webrtc.*
import java.util.concurrent.Executors
import javax.inject.Inject
class CallManager(private val context: Context,
private val storage: Storage): PeerConnection.Observer {
class CallManager(private val context: Context): PeerConnection.Observer,
SignalAudioManager.EventListener,
CallDataListener {
enum class CallState {
STATE_IDLE, STATE_DIALING, STATE_ANSWERING, STATE_REMOTE_RINGING, STATE_LOCAL_RINGING, STATE_CONNECTED
}
val signalAudioManager: SignalAudioManager by lazy {
SignalAudioManager(context, this, CallComponent.get(context).audioManagerCompat())
}
private val serviceExecutor = Executors.newSingleThreadExecutor()
private val networkExecutor = Executors.newSingleThreadExecutor()
private val eglBase: EglBase = EglBase.create()
private val connectionFactory by lazy {
private var peerConnectionWrapper: PeerConnectionWrapper? = null
val decoderFactory = DefaultVideoDecoderFactory(eglBase.eglBaseContext)
val encoderFactory = DefaultVideoEncoderFactory(eglBase.eglBaseContext, true, true)
private val currentCallState: MutableStateFlow<CallState> = MutableStateFlow(CallState.STATE_IDLE)
PeerConnectionFactory.builder()
.setVideoDecoderFactory(decoderFactory)
.setVideoEncoderFactory(encoderFactory)
.setOptions(PeerConnectionFactory.Options())
.createPeerConnectionFactory()!!
private fun createCameraCapturer(enumerator: CameraEnumerator): CameraVideoCapturer? {
val deviceNames = enumerator.deviceNames
// First, try to find front facing camera
Log.d("Loki-RTC-vid", "Looking for front facing cameras.")
for (deviceName in deviceNames) {
if (enumerator.isFrontFacing(deviceName)) {
Log.d("Loki-RTC-vid", "Creating front facing camera capturer.")
val videoCapturer = enumerator.createCapturer(deviceName, null)
if (videoCapturer != null) {
return videoCapturer
}
}
}
// Front facing camera not found, try something else
Log.d("Loki-RTC-vid", "Looking for other cameras.")
for (deviceName in deviceNames) {
if (!enumerator.isFrontFacing(deviceName)) {
Log.d("Loki-RTC-vid", "Creating other camera capturer.")
val videoCapturer = enumerator.createCapturer(deviceName, null)
if (videoCapturer != null) {
return videoCapturer
}
}
}
return null
}
private var peerConnection: PeerConnection? = null
override fun newCallMessage(callMessage: SignalServiceProtos.CallMessage) {
private fun getPeerConnection(): PeerConnection {
val stun = PeerConnection.IceServer.builder("stun:freyr.getsession.org:5349").setTlsCertPolicy(PeerConnection.TlsCertPolicy.TLS_CERT_POLICY_INSECURE_NO_CHECK).createIceServer()
val turn = PeerConnection.IceServer.builder("turn:freyr.getsession.org:5349").setUsername("webrtc").setPassword("webrtc").setTlsCertPolicy(PeerConnection.TlsCertPolicy.TLS_CERT_POLICY_INSECURE_NO_CHECK).createIceServer()
val iceServers = mutableListOf(turn, stun)
val rtcConfig = PeerConnection.RTCConfiguration(iceServers).apply {
this.tcpCandidatePolicy = PeerConnection.TcpCandidatePolicy.ENABLED
this.candidateNetworkPolicy = PeerConnection.CandidateNetworkPolicy.ALL
// this.iceTransportsType = PeerConnection.IceTransportsType.RELAY
}
rtcConfig.keyType = PeerConnection.KeyType.ECDSA
return connectionFactory.createPeerConnection(rtcConfig, this)!!
}
fun networkChange(networkAvailable: Boolean) {
@ -54,9 +82,11 @@ class CallManager(private val context: Context,
}
fun callEnded() {
peerConnection?.close()
peerConnection = null
peerConnectionWrapper?.()
peerConnectionWrapper = null
}
fun setAudioEnabled(isEnabled: Boolean) {
@ -110,4 +140,16 @@ class CallManager(private val context: Context,
override fun onAddTrack(p0: RtpReceiver?, p1: Array<out MediaStream>?) {
TODO("Not yet implemented")
}
override fun onAudioDeviceChanged(activeDevice: SignalAudioManager.AudioDevice, devices: Set<SignalAudioManager.AudioDevice>) {
TODO("Not yet implemented")
}
private fun CallMessage.iceCandidates(): List<IceCandidate> {
val candidateSize = sdpMids.size
return (0 until candidateSize).map { i ->
IceCandidate(sdpMids[i], sdpMLineIndexes[i], sdps[i])
}
}
}

View File

@ -6,23 +6,29 @@ import dagger.hilt.android.lifecycle.HiltViewModel
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.launchIn
import kotlinx.coroutines.flow.onEach
import org.session.libsession.messaging.messages.control.CallMessage
import org.webrtc.*
import javax.inject.Inject
@HiltViewModel
class CallViewModel @Inject constructor(
private val callManager: CallManager
): ViewModel(), PeerConnection.Observer {
class CallViewModel @Inject constructor(private val callManager: CallManager): ViewModel() {
sealed class StateEvent {
data class AudioEnabled(val isEnabled: Boolean): StateEvent()
data class VideoEnabled(val isEnabled: Boolean): StateEvent()
}
private val audioEnabledState = MutableStateFlow(StateEvent.AudioEnabled(true))
private val videoEnabledState = MutableStateFlow(StateEvent.VideoEnabled(false))
val audioEnabledState = MutableStateFlow(
callManager.audioEnabled.let { isEnabled ->
}
)
val videoEnabledState = MutableStateFlow(
callManager.videoEnabled.let { isEnabled ->
}
)
private val peerConnection = callManager.getPeerConnection(this)
// set up listeners for establishing connection toggling video / audio
init {
@ -32,48 +38,4 @@ class CallViewModel @Inject constructor(
.launchIn(viewModelScope)
}
override fun onSignalingChange(p0: PeerConnection.SignalingState?) {
}
override fun onIceConnectionChange(p0: PeerConnection.IceConnectionState?) {
}
override fun onIceConnectionReceivingChange(p0: Boolean) {
}
override fun onIceGatheringChange(p0: PeerConnection.IceGatheringState?) {
}
override fun onIceCandidate(p0: IceCandidate?) {
}
override fun onIceCandidatesRemoved(p0: Array<out IceCandidate>?) {
}
override fun onAddStream(p0: MediaStream?) {
}
override fun onRemoveStream(p0: MediaStream?) {
}
override fun onDataChannel(p0: DataChannel?) {
}
override fun onRenegotiationNeeded() {
}
override fun onAddTrack(p0: RtpReceiver?, p1: Array<out MediaStream>?) {
}
}

View File

@ -0,0 +1,75 @@
package org.thoughtcrime.securesms.webrtc
import android.content.Context
import org.thoughtcrime.securesms.webrtc.video.Camera
import org.thoughtcrime.securesms.webrtc.video.CameraEventListener
import org.webrtc.*
class PeerConnectionWrapper(context: Context,
factory: PeerConnectionFactory,
observer: PeerConnection.Observer,
localRenderer: VideoSink,
cameraEventListener: CameraEventListener,
eglBase: EglBase,
relay: Boolean = false) {
private val peerConnection: PeerConnection
private val audioTrack: AudioTrack
private val audioSource: AudioSource
private val camera: Camera
private val videoSource: VideoSource?
private val videoTrack: VideoTrack?
init {
val stun = PeerConnection.IceServer.builder("stun:freyr.getsession.org:5349").createIceServer()
val turn = PeerConnection.IceServer.builder("turn:freyr.getsession.org:5349").setUsername("webrtc").setPassword("webrtc").createIceServer()
val iceServers = listOf(stun,turn)
val constraints = MediaConstraints().apply {
optional.add(MediaConstraints.KeyValuePair("DtlsSrtpKeyAgreement", "true"))
}
val audioConstraints = MediaConstraints().apply {
optional.add(MediaConstraints.KeyValuePair("DtlsSrtpKeyAgreement", "true"))
}
val configuration = PeerConnection.RTCConfiguration(iceServers).apply {
bundlePolicy = PeerConnection.BundlePolicy.MAXBUNDLE
rtcpMuxPolicy = PeerConnection.RtcpMuxPolicy.REQUIRE
if (relay) {
iceTransportsType = PeerConnection.IceTransportsType.RELAY
}
}
peerConnection = factory.createPeerConnection(configuration, constraints, observer)!!
peerConnection.setAudioPlayout(false)
peerConnection.setAudioRecording(false)
val mediaStream = factory.createLocalMediaStream("ARDAMS")
audioSource = factory.createAudioSource(audioConstraints)
audioTrack = factory.createAudioTrack("ARDAMSa0", audioSource)
audioTrack.setEnabled(false)
mediaStream.addTrack(audioTrack)
camera = Camera(context, cameraEventListener)
if (camera.capturer != null) {
videoSource = factory.createVideoSource(false)
videoTrack = factory.createVideoTrack("ARDAMSv0", videoSource)
camera.capturer.initialize(
SurfaceTextureHelper.create("WebRTC-SurfaceTextureHelper", eglBase.eglBaseContext),
context,
videoSource.capturerObserver
)
videoTrack.addSink(localRenderer)
videoTrack.setEnabled(false)
mediaStream.addTrack(videoTrack)
} else {
videoSource = null
videoTrack = null
}
peerConnection.addStream(mediaStream)
}
}

View File

@ -2,10 +2,10 @@ package org.thoughtcrime.securesms.webrtc.data
import org.session.libsession.database.CallDataProvider
import org.session.libsession.database.StorageProtocol
import org.thoughtcrime.securesms.webrtc.CallManager
import javax.inject.Inject
class SessionCallDataProvider @Inject constructor(private val storage: StorageProtocol): CallDataProvider {
class SessionCallDataProvider @Inject constructor(private val storage: StorageProtocol,
private val callManager: CallManager): CallDataProvider {
}

View File

@ -0,0 +1,74 @@
package org.thoughtcrime.securesms.webrtc.video
import android.content.Context
import org.session.libsignal.utilities.Log
import org.thoughtcrime.securesms.webrtc.video.CameraState.Direction.*
import org.webrtc.Camera2Enumerator
import org.webrtc.CameraEnumerator
import org.webrtc.CameraVideoCapturer
class Camera(context: Context,
private val cameraEventListener: CameraEventListener): CameraVideoCapturer.CameraSwitchHandler {
companion object {
private val TAG = Log.tag(Camera::class.java)
}
val capturer: CameraVideoCapturer?
private val cameraCount: Int
private var activeDirection: CameraState.Direction = PENDING
var enabled: Boolean = false
set(value) {
field = value
capturer ?: return
try {
if (value) {
capturer.startCapture(1280,720,30)
} else {
capturer.stopCapture()
}
} catch (e: InterruptedException) {
Log.e(TAG,"Interrupted while stopping video capture")
}
}
init {
val enumerator = Camera2Enumerator(context)
cameraCount = enumerator.deviceNames.size
capturer = createVideoCapturer(enumerator, FRONT)?.apply {
activeDirection = FRONT
} ?: createVideoCapturer(enumerator, BACK)?.apply {
activeDirection = BACK
} ?: run {
activeDirection = NONE
null
}
}
fun flip() {
if (capturer == null || cameraCount < 2) {
Log.w(TAG, "Tried to flip camera without capturer or less than 2 cameras")
return
}
activeDirection = PENDING
capturer.switchCamera(this)
}
override fun onCameraSwitchDone(isFrontFacing: Boolean) {
activeDirection = if (isFrontFacing) FRONT else BACK
cameraEventListener.onCameraSwitchCompleted(CameraState(activeDirection, cameraCount))
}
override fun onCameraSwitchError(errorMessage: String?) {
Log.e(TAG,"onCameraSwitchError: $errorMessage")
cameraEventListener.onCameraSwitchCompleted(CameraState(activeDirection, cameraCount))
}
private fun createVideoCapturer(enumerator: CameraEnumerator, direction: CameraState.Direction): CameraVideoCapturer? =
enumerator.deviceNames.firstOrNull { device ->
(direction == FRONT && enumerator.isFrontFacing(device)) ||
(direction == BACK && enumerator.isBackFacing(device))
}?.let { enumerator.createCapturer(it, null) }
}

View File

@ -0,0 +1,18 @@
package org.thoughtcrime.securesms.webrtc.video
interface CameraEventListener {
fun onCameraSwitchCompleted(newCameraState: CameraState)
}
data class CameraState(val activeDirection: Direction, val cameraCount: Int) {
companion object {
val UNKNOWN = CameraState(Direction.NONE, 0)
}
val enabled: Boolean
get() = activeDirection != Direction.NONE
enum class Direction {
FRONT, BACK, NONE, PENDING
}
}

View File

@ -51,6 +51,6 @@ allprojects {
project.ext {
androidMinimumSdkVersion = 23
androidCompileSdkVersion = 30
androidCompileSdkVersion = 31
}
}