feat: adding more command handlers in WebRtcCallService.kt

This commit is contained in:
jubb 2021-11-04 17:14:07 +11:00
parent 5cff5ffb45
commit de4d8e9be4
7 changed files with 471 additions and 23 deletions

View File

@ -19,6 +19,7 @@ import org.session.libsession.utilities.Util
import org.session.libsession.utilities.recipients.Recipient
import org.session.libsignal.utilities.Log
import org.thoughtcrime.securesms.calls.WebRtcCallActivity
import org.thoughtcrime.securesms.dependencies.DatabaseComponent
import org.thoughtcrime.securesms.util.CallNotificationBuilder
import org.thoughtcrime.securesms.util.CallNotificationBuilder.Companion.TYPE_ESTABLISHED
import org.thoughtcrime.securesms.util.CallNotificationBuilder.Companion.TYPE_INCOMING_CONNECTING
@ -79,8 +80,6 @@ class WebRtcCallService: Service() {
const val EXTRA_ICE_SDP_LINE_INDEX = "ice_sdp_line_index"
const val EXTRA_RESULT_RECEIVER = "result_receiver"
const val DATA_CHANNEL_NAME = "signaling"
const val INVALID_NOTIFICATION_ID = -1
fun acceptCallIntent(context: Context) = Intent(context, WebRtcCallService::class.java).setAction(ACTION_ANSWER_CALL)
@ -109,6 +108,7 @@ class WebRtcCallService: Service() {
private var lastNotification: Notification? = null
private val serviceExecutor = Executors.newSingleThreadExecutor()
private val timeoutExecutor = Executors.newScheduledThreadPool(1)
private val hangupOnCallAnswered = HangUpRtcOnPstnCallAnsweredListener {
startService(hangupIntent(this))
}
@ -163,7 +163,6 @@ class WebRtcCallService: Service() {
override fun onCreate() {
super.onCreate()
callManager.initializeResources(this)
// create audio manager
registerIncomingPstnCallReceiver()
registerWiredHeadsetStateReceiver()
@ -230,15 +229,147 @@ class WebRtcCallService: Service() {
private fun handleIncomingCall(intent: Intent) {
if (callManager.currentConnectionState != STATE_IDLE) throw IllegalStateException("Incoming on non-idle")
val offer = intent.getStringExtra(EXTRA_REMOTE_DESCRIPTION)
val offer = intent.getStringExtra(EXTRA_REMOTE_DESCRIPTION) ?: return
callManager.postConnectionEvent(STATE_ANSWERING)
callManager.callId = getCallId(intent)
val callId = getCallId(intent)
callManager.callId = callId
callManager.clearPendingIceUpdates()
val recipient = getRemoteRecipient(intent)
callManager.recipient = recipient
if (isIncomingMessageExpired(intent)) {
insertMissedCall(recipient, true)
terminate()
return
}
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
setCallInProgressNotification(TYPE_INCOMING_CONNECTING, recipient)
}
timeoutExecutor.schedule(TimeoutRunnable(callId, this), 2, TimeUnit.MINUTES)
callManager.initializeVideo(this)
val expectedState = callManager.currentConnectionState
val expectedCallId = callManager.callId
try {
val answerFuture = callManager.onIncomingCall(offer, this) // add is always turn here
answerFuture.fail { e ->
if (isConsistentState(expectedState,expectedCallId, callManager.currentConnectionState, callManager.callId)) {
Log.e(TAG, e)
insertMissedCall(recipient, true)
terminate()
}
}
callManager.postViewModelState(CallViewModel.State.CALL_INCOMING)
// lock manager update phone state processing here
} catch (e: Exception) {
Log.e(TAG,e)
terminate()
}
}
private fun handleOutgoingCall(intent: Intent) {
if (callManager.currentConnectionState != STATE_IDLE) throw IllegalStateException("Dialing from non-idle")
callManager.postConnectionEvent(STATE_DIALING)
callManager.recipient = getRemoteRecipient(intent)
val callId = UUID.randomUUID()
callManager.callId = callId
callManager.initializeVideo(this)
callManager.postViewModelState(CallViewModel.State.CALL_OUTGOING)
// update phone state IN_CALL
callManager.initializeAudioForCall()
callManager.startOutgoingRinger(OutgoingRinger.Type.RINGING)
// bluetoothStateManager.setWantsConnection(true)
setCallInProgressNotification(TYPE_OUTGOING_RINGING, callManager.recipient)
// DatabaseComponent.get(this).insertOutgoingCall(callManager.recipient!!.address)
timeoutExecutor.schedule(TimeoutRunnable(callId, this), 2, TimeUnit.MINUTES)
val expectedState = callManager.currentConnectionState
val expectedCallId = callManager.callId
try {
val offerFuture = callManager.onOutgoingCall(this)
offerFuture.fail { e ->
if (isConsistentState(expectedState, expectedCallId, callManager.currentConnectionState, callManager.callId)) {
Log.e(TAG,e)
callManager.postViewModelState(CallViewModel.State.NETWORK_FAILURE)
terminate()
}
}
} catch (e: Exception) {
Log.e(TAG,e)
terminate()
}
}
private fun handleAnswerCall(intent: Intent) {
if (callManager.currentConnectionState != STATE_LOCAL_RINGING) {
Log.e(TAG,"Can only answer from ringing!")
return
}
if (callManager.callNotSetup()) {
throw AssertionError("assert")
}
// DatabaseComponent.get(this).smsDatabase().insertReceivedCall(recipient)
val (callId, recipient) = callManager.handleAnswerCall()
intent.putExtra(EXTRA_CALL_ID, callId)
intent.putExtra(EXTRA_RECIPIENT_ADDRESS, recipient.address)
handleCallConnected(intent)
}
private fun handleDenyCall(intent: Intent) {
if (callManager.currentConnectionState != STATE_LOCAL_RINGING) {
Log.e(TAG,"Can only deny from ringing!")
return
}
if (callManager.callNotSetup()) {
throw AssertionError("assert")
}
callManager.handleDenyCall()
// DatabaseComponent.get(this).smsDatabase().insertMissedCall(recipient)
terminate()
}
private fun handleLocalHangup(intent: Intent) {
callManager.handleLocalHangup()
terminate()
}
private fun handleRemoteHangup(intent: Intent) {
if (callManager.callId != getCallId(intent)) {
Log.e(TAG, "Hangup for non-active call...")
return
}
callManager.handleRemoteHangup()
if (callManager.currentConnectionState in arrayOf(STATE_ANSWERING, STATE_LOCAL_RINGING)) {
callManager.recipient?.let { recipient ->
insertMissedCall(recipient, true)
}
}
}
private fun handleSetMuteAudio(intent: Intent) {
val muted = intent.getBooleanExtra(EXTRA_MUTE, false)
callManager.handleSetMuteAudio(muted)
}
private fun handleSetMuteVideo(intent: Intent) {
val muted = intent.getBooleanExtra(EXTRA_MUTE, false)
callManager.handleSetMuteVideo(muted)
}
private fun handleCheckTimeout(intent: Intent) {
@ -300,10 +431,27 @@ class WebRtcCallService: Service() {
}
}
private abstract class FailureListener<V>(
expectedState: CallManager.CallState,
expectedCallId: UUID?,
getState: () -> Pair<CallManager.CallState, UUID?>): StateAwareListener<V>(expectedState, expectedCallId, getState) {
override fun onSuccessContinue(result: V) {}
}
private abstract class SuccessOnlyListener<V>(
expectedState: CallManager.CallState,
expectedCallId: UUID?,
getState: () -> Pair<CallManager.CallState, UUID>): StateAwareListener<V>(expectedState, expectedCallId, getState) {
override fun onFailureContinue(throwable: Throwable?) {
Log.e(TAG, throwable)
throw AssertionError(throwable)
}
}
private abstract class StateAwareListener<V>(
private val expectedState: CallManager.CallState,
private val expectedCallId: UUID,
private val getState: ()->Pair<CallManager.CallState, UUID>): FutureTaskListener<V> {
private val expectedCallId: UUID?,
private val getState: ()->Pair<CallManager.CallState, UUID?>): FutureTaskListener<V> {
companion object {
private val TAG = Log.tag(StateAwareListener::class.java)
@ -338,4 +486,13 @@ class WebRtcCallService: Service() {
}
private fun isConsistentState(
expectedState: CallManager.CallState,
expectedCallId: UUID?,
currentState: CallManager.CallState,
currentCallId: UUID?
): Boolean {
return expectedState == currentState && expectedCallId == currentCallId
}
}

View File

@ -1,26 +1,30 @@
package org.thoughtcrime.securesms.webrtc
import android.content.Context
import android.content.Intent
import android.telephony.TelephonyManager
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.asSharedFlow
import nl.komponents.kovenant.Promise
import org.session.libsession.messaging.messages.control.CallMessage
import org.session.libsession.messaging.sending_receiving.MessageSender
import org.session.libsession.utilities.Util
import org.session.libsession.utilities.recipients.Recipient
import org.session.libsignal.protos.SignalServiceProtos
import org.session.libsignal.utilities.Log
import org.thoughtcrime.securesms.service.WebRtcCallService
import org.thoughtcrime.securesms.webrtc.audio.AudioManagerCompat
import org.thoughtcrime.securesms.webrtc.audio.OutgoingRinger
import org.thoughtcrime.securesms.webrtc.audio.SignalAudioManager
import org.thoughtcrime.securesms.webrtc.video.CameraEventListener
import org.thoughtcrime.securesms.webrtc.video.CameraState
import org.webrtc.*
import java.lang.NullPointerException
import java.util.*
import java.util.concurrent.Executors
class CallManager(context: Context, audioManager: AudioManagerCompat): PeerConnection.Observer,
SignalAudioManager.EventListener,
CallDataListener {
CallDataListener, CameraEventListener, DataChannel.Observer {
enum class CallState {
STATE_IDLE, STATE_DIALING, STATE_ANSWERING, STATE_REMOTE_RINGING, STATE_LOCAL_RINGING, STATE_CONNECTED
@ -47,6 +51,7 @@ class CallManager(context: Context, audioManager: AudioManagerCompat): PeerConne
CallState.STATE_CONNECTED
)
val DISCONNECTED_STATES = arrayOf(CallState.STATE_IDLE)
private const val DATA_CHANNEL_NAME = "signaling"
}
@ -63,8 +68,6 @@ class CallManager(context: Context, audioManager: AudioManagerCompat): PeerConne
private val _callStateEvents = MutableStateFlow(CallViewModel.State.CALL_PENDING)
val callStateEvents = _callStateEvents.asSharedFlow()
private var localCameraState: CameraState = CameraState.UNKNOWN
private var microphoneEnabled = true
private var remoteVideoEnabled = false
private var bluetoothAvailable = false
val currentConnectionState = (_connectionEvents.value as StateEvent.CallStateUpdate).state
@ -75,7 +78,10 @@ class CallManager(context: Context, audioManager: AudioManagerCompat): PeerConne
var callId: UUID? = null
var recipient: Recipient? = null
private var peerConnectionWrapper: PeerConnectionWrapper? = null
fun getCurrentCallState(): Pair<CallState, UUID?> = currentConnectionState to callId
private var peerConnection: PeerConnectionWrapper? = null
private var dataChannel: DataChannel? = null
private val pendingOutgoingIceUpdates = ArrayDeque<IceCandidate>()
@ -90,6 +96,10 @@ class CallManager(context: Context, audioManager: AudioManagerCompat): PeerConne
pendingIncomingIceUpdates.clear()
}
fun initializeAudioForCall() {
signalAudioManager.initializeAudioForCall()
}
fun startOutgoingRinger(ringerType: OutgoingRinger.Type) {
signalAudioManager.startOutgoingRinger(ringerType)
}
@ -177,20 +187,20 @@ class CallManager(context: Context, audioManager: AudioManagerCompat): PeerConne
}
fun callEnded() {
peerConnectionWrapper?.dispose()
peerConnectionWrapper = null
peerConnection?.dispose()
peerConnection = null
}
fun setAudioEnabled(isEnabled: Boolean) {
currentConnectionState.withState(*(CONNECTED_STATES + PENDING_CONNECTION_STATES)) {
peerConnectionWrapper?.setAudioEnabled(isEnabled)
peerConnection?.setAudioEnabled(isEnabled)
_audioEvents.value = StateEvent.AudioEnabled(true)
}
}
fun setVideoEnabled(isEnabled: Boolean) {
currentConnectionState.withState(*(CONNECTED_STATES + PENDING_CONNECTION_STATES)) {
peerConnectionWrapper?.setVideoEnabled(isEnabled)
peerConnection?.setVideoEnabled(isEnabled)
_audioEvents.value = StateEvent.AudioEnabled(true)
}
}
@ -239,6 +249,19 @@ class CallManager(context: Context, audioManager: AudioManagerCompat): PeerConne
}
override fun onBufferedAmountChange(l: Long) {
Log.i(TAG,"onBufferedAmountChange: $l")
}
override fun onStateChange() {
Log.i(TAG,"onStateChange")
}
override fun onMessage(buffer: DataChannel.Buffer?) {
Log.i(TAG,"onMessage...")
TODO("interpret the data channel buffer and check for signals")
}
override fun onAudioDeviceChanged(activeDevice: SignalAudioManager.AudioDevice, devices: Set<SignalAudioManager.AudioDevice>) {
signalAudioManager.handleCommand(AudioManagerCommand())
}
@ -250,15 +273,15 @@ class CallManager(context: Context, audioManager: AudioManagerCompat): PeerConne
}
}
private fun CallState.withState(vararg expected: CallState, transition: ()->Unit) {
private fun CallState.withState(vararg expected: CallState, transition: () -> Unit) {
if (this in expected) transition()
else Log.w(TAG,"Tried to transition state $this but expected $expected")
}
fun stop() {
signalAudioManager.stop(currentConnectionState in OUTGOING_STATES)
peerConnectionWrapper?.dispose()
peerConnectionWrapper = null
peerConnection?.dispose()
peerConnection = null
localRenderer?.release()
remoteRenderer?.release()
@ -278,8 +301,119 @@ class CallManager(context: Context, audioManager: AudioManagerCompat): PeerConne
pendingIncomingIceUpdates.clear()
}
fun initializeResources(webRtcCallService: WebRtcCallService) {
TODO("Not yet implemented")
override fun onCameraSwitchCompleted(newCameraState: CameraState) {
localCameraState = newCameraState
}
fun onIncomingCall(offer: String, context: Context, isAlwaysTurn: Boolean = false): Promise<Unit, Exception> {
val callId = callId ?: return Promise.ofFail(NullPointerException("callId is null"))
val recipient = recipient ?: return Promise.ofFail(NullPointerException("recipient is null"))
val factory = peerConnectionFactory ?: return Promise.ofFail(NullPointerException("peerConnectionFactory is null"))
val local = localRenderer ?: return Promise.ofFail(NullPointerException("localRenderer is null"))
val base = eglBase ?: return Promise.ofFail(NullPointerException("eglBase is null"))
val connection = PeerConnectionWrapper(
context,
factory,
this,
local,
this,
base,
isAlwaysTurn
)
peerConnection = connection
localCameraState = connection.getCameraState()
connection.setRemoteDescription(SessionDescription(SessionDescription.Type.OFFER, offer))
val answer = connection.createAnswer(MediaConstraints())
connection.setLocalDescription(answer)
val answerMessage = MessageSender.sendNonDurably(CallMessage.answer(
answer.description,
callId
), recipient.address)
while (pendingIncomingIceUpdates.isNotEmpty()) {
val candidate = pendingIncomingIceUpdates.pop() ?: break
connection.addIceCandidate(candidate)
}
return answerMessage // TODO: maybe add success state update
}
fun onOutgoingCall(context: Context, isAlwaysTurn: Boolean = false): Promise<Unit, Exception> {
val callId = callId ?: return Promise.ofFail(NullPointerException("callId is null"))
val recipient = recipient
?: return Promise.ofFail(NullPointerException("recipient is null"))
val factory = peerConnectionFactory
?: return Promise.ofFail(NullPointerException("peerConnectionFactory is null"))
val local = localRenderer
?: return Promise.ofFail(NullPointerException("localRenderer is null"))
val base = eglBase ?: return Promise.ofFail(NullPointerException("eglBase is null"))
val connection = PeerConnectionWrapper(
context,
factory,
this,
local,
this,
base,
isAlwaysTurn
)
localCameraState = connection.getCameraState()
val dataChannel = connection.createDataChannel(DATA_CHANNEL_NAME)
dataChannel.registerObserver(this)
val offer = connection.createOffer(MediaConstraints())
connection.setLocalDescription(offer)
Log.i(TAG, "Sending offer: ${offer.description}")
return MessageSender.sendNonDurably(CallMessage.offer(
offer.description,
callId
), recipient.address)
}
fun callNotSetup(): Boolean =
peerConnection == null || dataChannel == null || recipient == null || callId == null
fun handleAnswerCall(): Pair<UUID, Recipient> {
peerConnection?.let { connection ->
connection.setAudioEnabled(true)
connection.setVideoEnabled(true)
}
return callId!! to recipient!!
}
fun handleDenyCall() {
val callId = callId ?: return
val recipient = recipient ?: return
MessageSender.sendNonDurably(CallMessage.endCall(callId), recipient.address)
}
fun handleLocalHangup() {
val recipient = recipient ?: return
val callId = callId ?: return
postViewModelState(CallViewModel.State.CALL_DISCONNECTED)
MessageSender.sendNonDurably(CallMessage.endCall(callId), recipient.address)
}
fun handleRemoteHangup() {
when (currentConnectionState) {
CallState.STATE_DIALING,
CallState.STATE_REMOTE_RINGING -> postViewModelState(CallViewModel.State.RECIPIENT_UNAVAILABLE)
else -> postViewModelState(CallViewModel.State.CALL_DISCONNECTED)
}
}
fun handleSetMuteAudio(muted: Boolean) {
_audioEvents.value = StateEvent.AudioEnabled(!muted)
peerConnection?.setAudioEnabled(_audioEvents.value.isEnabled)
}
fun handleSetMuteVideo(muted: Boolean) {
_videoEvents.value = StateEvent.VideoEnabled(!muted)
peerConnection?.setVideoEnabled(_videoEvents.value.isEnabled)
TODO()
}
}

View File

@ -0,0 +1,6 @@
package org.thoughtcrime.securesms.webrtc
class PeerConnectionException: Exception {
constructor(error: String?): super(error)
constructor(throwable: Throwable): super(throwable)
}

View File

@ -1,9 +1,13 @@
package org.thoughtcrime.securesms.webrtc
import android.content.Context
import kotlinx.coroutines.runBlocking
import org.session.libsignal.utilities.SettableFuture
import org.thoughtcrime.securesms.webrtc.video.Camera
import org.thoughtcrime.securesms.webrtc.video.CameraEventListener
import org.thoughtcrime.securesms.webrtc.video.CameraState
import org.webrtc.*
import java.util.concurrent.ExecutionException
class PeerConnectionWrapper(context: Context,
factory: PeerConnectionFactory,
@ -72,6 +76,14 @@ class PeerConnectionWrapper(context: Context,
peerConnection.addStream(mediaStream)
}
fun getCameraState(): CameraState {
return CameraState(camera.activeDirection, camera.cameraCount)
}
fun createDataChannel(channelName: String): DataChannel {
}
fun addIceCandidate(candidate: IceCandidate) {
// TODO: filter logic based on known servers
peerConnection.addIceCandidate(candidate)
@ -87,6 +99,124 @@ class PeerConnectionWrapper(context: Context,
peerConnection.dispose()
}
fun setRemoteDescription(description: SessionDescription) {
val future = SettableFuture<Boolean>()
peerConnection.setRemoteDescription(object: SdpObserver {
override fun onCreateSuccess(p0: SessionDescription?) {
throw AssertionError()
}
override fun onCreateFailure(p0: String?) {
throw AssertionError()
}
override fun onSetSuccess() {
future.set(true)
}
override fun onSetFailure(error: String?) {
future.setException(PeerConnectionException(error))
}
}, description)
try {
future.get()
} catch (e: InterruptedException) {
throw AssertionError(e)
} catch (e: ExecutionException) {
throw PeerConnectionException(e)
}
}
fun createAnswer(mediaConstraints: MediaConstraints) : SessionDescription {
val future = SettableFuture<SessionDescription>()
peerConnection.createAnswer(object:SdpObserver {
override fun onCreateSuccess(sdp: SessionDescription?) {
future.set(sdp)
}
override fun onSetSuccess() {
throw AssertionError()
}
override fun onCreateFailure(p0: String?) {
future.setException(PeerConnectionException(p0))
}
override fun onSetFailure(p0: String?) {
throw AssertionError()
}
}, mediaConstraints)
try {
return future.get()
} catch (e: InterruptedException) {
throw AssertionError()
} catch (e: ExecutionException) {
throw PeerConnectionException(e)
}
}
fun createOffer(mediaConstraints: MediaConstraints): SessionDescription {
val future = SettableFuture<SessionDescription>()
peerConnection.createAnswer(object:SdpObserver {
override fun onCreateSuccess(sdp: SessionDescription?) {
future.set(sdp)
}
override fun onSetSuccess() {
throw AssertionError()
}
override fun onCreateFailure(p0: String?) {
future.setException(PeerConnectionException(p0))
}
override fun onSetFailure(p0: String?) {
throw AssertionError()
}
}, mediaConstraints)
try {
return future.get()
} catch (e: InterruptedException) {
throw AssertionError()
} catch (e: ExecutionException) {
throw PeerConnectionException(e)
}
}
fun setLocalDescription(sdp: SessionDescription) {
val future = SettableFuture<Boolean>()
peerConnection.setLocalDescription(object: SdpObserver {
override fun onCreateSuccess(p0: SessionDescription?) {
}
override fun onSetSuccess() {
future.set(true)
}
override fun onCreateFailure(p0: String?) {}
override fun onSetFailure(error: String?) {
future.setException(PeerConnectionException(error))
}
}, sdp)
try {
future.get()
} catch(e: InterruptedException) {
throw AssertionError(e)
} catch(e: ExecutionException) {
throw PeerConnectionException(e)
}
}
fun setAudioEnabled(isEnabled: Boolean) {
audioTrack.setEnabled(isEnabled)
}

View File

@ -4,12 +4,14 @@ import android.content.BroadcastReceiver
import android.content.Context
import android.content.Intent
import android.content.IntentFilter
import android.media.AudioFocusRequest
import android.media.AudioManager
import android.media.SoundPool
import android.net.Uri
import android.os.Build
import android.os.HandlerThread
import network.loki.messenger.R
import org.session.libsession.utilities.ServiceUtil
import org.session.libsession.utilities.concurrent.SignalExecutors
import org.session.libsignal.utilities.Log
import org.thoughtcrime.securesms.webrtc.AudioManagerCommand
@ -355,6 +357,11 @@ class SignalAudioManager(private val context: Context,
updateAudioDeviceState()
}
fun initializeAudioForCall() {
val audioManager: AudioManager = ServiceUtil.getAudioManager(context)
audioManager.requestAudioFocus(null, AudioManager.STREAM_VOICE_CALL, AudioManager.AUDIOFOCUS_GAIN_TRANSIENT_EXCLUSIVE)
}
private inner class WiredHeadsetReceiver : BroadcastReceiver() {
override fun onReceive(context: Context, intent: Intent) {
val pluggedIn = intent.getIntExtra("state", 0) == 1

View File

@ -15,8 +15,8 @@ class Camera(context: Context,
}
val capturer: CameraVideoCapturer?
private val cameraCount: Int
private var activeDirection: CameraState.Direction = PENDING
val cameraCount: Int
var activeDirection: CameraState.Direction = PENDING
var enabled: Boolean = false
set(value) {
field = value

View File

@ -33,6 +33,20 @@ class CallMessage(): ControlMessage() {
companion object {
const val TAG = "CallMessage"
fun answer(sdp: String, callId: UUID) = CallMessage(SignalServiceProtos.CallMessage.Type.ANSWER,
listOf(sdp),
listOf(),
listOf(),
callId
)
fun offer(sdp: String, callId: UUID) = CallMessage(SignalServiceProtos.CallMessage.Type.OFFER,
listOf(sdp),
listOf(),
listOf(),
callId
)
fun endCall(callId: UUID) = CallMessage(SignalServiceProtos.CallMessage.Type.END_CALL, emptyList(), emptyList(), emptyList(), callId)
fun fromProto(proto: SignalServiceProtos.Content): CallMessage? {