Merge pull request #2027 from Bilb/fix-netwokr-switching

Fix network switching with ongoing webrtc calls

_locales/en/messages.json

@@ -444,7 +444,7 @@
   "endCall": "End call",
   "cameraPermissionNeededTitle": "Voice/Video Call permissions required",
   "cameraPermissionNeeded": "You can enable the 'Voice and video calls' permission in the Privacy Settings.",
-  "unableToCall": "cancel your ongoing call first",
+  "unableToCall": "Cancel your ongoing call first",
   "unableToCallTitle": "Cannot start new call",
   "callMissed": "Missed call from $name$",
   "callMissedTitle": "Call missed",

ts/components/session/calling/CallButtons.tsx

@@ -71,16 +71,16 @@ export const AudioInputButton = ({
 
 export const AudioOutputButton = ({
   currentConnectedAudioOutputs,
-  isAudioOutputMuted,
-  hideArrowIcon = false,
-}: {
+}: // isAudioOutputMuted,
+// hideArrowIcon = false,
+{
   currentConnectedAudioOutputs: Array<InputItem>;
   isAudioOutputMuted: boolean;
   hideArrowIcon?: boolean;
 }) => {
   return (
     <>
-      <DropDownAndToggleButton
+      {/* <DropDownAndToggleButton
         iconType="volume"
         isMuted={isAudioOutputMuted}
         onMainButtonClick={() => {
@@ -90,7 +90,7 @@ export const AudioOutputButton = ({
           showAudioOutputMenu(currentConnectedAudioOutputs, e);
         }}
         hidePopoverArrow={hideArrowIcon}
-      />
+      /> */}
 
       <AudioOutputMenu
         triggerId={audioOutputTriggerId}
@@ -238,19 +238,19 @@ const showAudioInputMenu = (
   });
 };
 
-const showAudioOutputMenu = (
-  currentConnectedAudioOutputs: Array<any>,
-  e: React.MouseEvent<HTMLDivElement>
-) => {
-  if (currentConnectedAudioOutputs.length === 0) {
-    ToastUtils.pushNoAudioOutputFound();
-    return;
-  }
-  contextMenu.show({
-    id: audioOutputTriggerId,
-    event: e,
-  });
-};
+// const showAudioOutputMenu = (
+//   currentConnectedAudioOutputs: Array<any>,
+//   e: React.MouseEvent<HTMLDivElement>
+// ) => {
+//   if (currentConnectedAudioOutputs.length === 0) {
+//     ToastUtils.pushNoAudioOutputFound();
+//     return;
+//   }
+//   contextMenu.show({
+//     id: audioOutputTriggerId,
+//     event: e,
+//   });
+// };
 
 const showVideoInputMenu = (
   currentConnectedCameras: Array<InputItem>,
@@ -300,22 +300,22 @@ const handleMicrophoneToggle = async (
   }
 };
 
-const handleSpeakerToggle = async (
-  currentConnectedAudioOutputs: Array<InputItem>,
-  isAudioOutputMuted: boolean
-) => {
-  if (!currentConnectedAudioOutputs.length) {
-    ToastUtils.pushNoAudioInputFound();
+// const handleSpeakerToggle = async (
+//   currentConnectedAudioOutputs: Array<InputItem>,
+//   isAudioOutputMuted: boolean
+// ) => {
+//   if (!currentConnectedAudioOutputs.length) {
+//     ToastUtils.pushNoAudioInputFound();
 
-    return;
-  }
-  if (isAudioOutputMuted) {
-    // selects the first one
-    await CallManager.selectAudioOutputByDeviceId(currentConnectedAudioOutputs[0].deviceId);
-  } else {
-    await CallManager.selectAudioOutputByDeviceId(CallManager.DEVICE_DISABLED_DEVICE_ID);
-  }
-};
+//     return;
+//   }
+//   if (isAudioOutputMuted) {
+//     // selects the first one
+//     await CallManager.selectAudioOutputByDeviceId(currentConnectedAudioOutputs[0].deviceId);
+//   } else {
+//     await CallManager.selectAudioOutputByDeviceId(CallManager.DEVICE_DISABLED_DEVICE_ID);
+//   }
+// };
 
 const StyledCallWindowControls = styled.div`
   position: absolute;

ts/components/session/calling/InConversationCallContainer.tsx

@@ -24,7 +24,6 @@ import { useModuloWithTripleDots } from '../../../hooks/useModuloWithTripleDots'
 import { CallWindowControls } from './CallButtons';
 import { SessionSpinner } from '../SessionSpinner';
 import { DEVICE_DISABLED_DEVICE_ID } from '../../../session/utils/CallManager';
-// import { useCallAudioLevel } from '../../../hooks/useCallAudioLevel';
 
 const VideoContainer = styled.div`
   height: 100%;
@@ -146,8 +145,6 @@ export const InConversationCallContainer = () => {
     isAudioOutputMuted,
   } = useVideoCallEventsListener('InConversationCallContainer', true);
 
-  // const isSpeaking = useCallAudioLevel();
-
   if (videoRefRemote?.current && videoRefLocal?.current) {
     if (videoRefRemote.current.srcObject !== remoteStream) {
       videoRefRemote.current.srcObject = remoteStream;
@@ -161,7 +158,7 @@ export const InConversationCallContainer = () => {
       if (currentSelectedAudioOutput === DEVICE_DISABLED_DEVICE_ID) {
         videoRefLocal.current.muted = true;
       } else {
-        void videoRefLocal.current.setSinkId(currentSelectedAudioOutput);
+        // void videoRefLocal.current.setSinkId(currentSelectedAudioOutput);
         videoRefLocal.current.muted = false;
       }
     }

ts/models/conversation.ts

@@ -176,7 +176,7 @@ export const fillConvoAttributesWithDefaults = (
   });
 };
 
-export type CallState = 'offering' | 'incoming' | 'connecting' | 'ongoing' | 'none' | undefined;
+export type CallState = 'offering' | 'incoming' | 'connecting' | 'ongoing' | undefined;
 
 export class ConversationModel extends Backbone.Model<ConversationAttributes> {
   public updateLastMessage: () => any;

ts/receiver/callMessage.ts

@@ -50,7 +50,7 @@ export async function handleCallMessage(
   if (type === SignalService.CallMessage.Type.END_CALL) {
     await removeFromCache(envelope);
 
-    CallManager.handleCallTypeEndCall(sender);
+    CallManager.handleCallTypeEndCall(sender, callMessage.uuid);
 
     return;
   }

ts/session/utils/CallManager.ts

@@ -22,16 +22,12 @@ import { PubKey } from '../types';
 
 import { v4 as uuidv4 } from 'uuid';
 import { PnServer } from '../../pushnotification';
-// import { SoundMeter } from '../../../ts/components/session/calling/SoundMeter';
 import { setIsRinging } from './RingingManager';
 
 export type InputItem = { deviceId: string; label: string };
 
 let currentCallUUID: string | undefined;
 
-// const VIDEO_WIDTH = 640;
-// const VIDEO_RATIO = 16 / 9;
-
 export type CallManagerOptionsType = {
   localStream: MediaStream | null;
   remoteStream: MediaStream | null;
@@ -280,7 +276,6 @@ export async function selectAudioInputByDeviceId(audioInputDeviceId: string) {
 export async function selectAudioOutputByDeviceId(audioOutputDeviceId: string) {
   if (audioOutputDeviceId === DEVICE_DISABLED_DEVICE_ID) {
     selectedAudioOutputId = audioOutputDeviceId;
-    console.warn('selectedAudioOutputId', selectedAudioOutputId);
 
     callVideoListeners();
     return;
@@ -288,13 +283,11 @@ export async function selectAudioOutputByDeviceId(audioOutputDeviceId: string) {
   if (audioOutputsList.some(m => m.deviceId === audioOutputDeviceId)) {
     selectedAudioOutputId = audioOutputDeviceId;
 
-    console.warn('selectedAudioOutputId', selectedAudioOutputId);
-
     callVideoListeners();
   }
 }
 
-async function handleNegotiationNeededEvent(_event: Event, recipient: string) {
+async function handleNegotiationNeededEvent(recipient: string) {
   try {
     makingOffer = true;
     window.log.info('got handleNegotiationNeeded event. creating offer');
@@ -476,12 +469,12 @@ const findLastMessageTypeFromSender = (sender: string, msgType: SignalService.Ca
   // FIXME this does not sort by timestamp as we do not have a timestamp stored in the SignalService.CallMessage object...
   const allMsg = _.flattenDeep([...msgCacheFromSenderWithDevices.values()]);
   const allMsgFromType = allMsg.filter(m => m.type === msgType);
-  const lastOfferMessage = _.last(allMsgFromType);
+  const lastMessageOfType = _.last(allMsgFromType);
 
-  if (!lastOfferMessage) {
+  if (!lastMessageOfType) {
     return undefined;
   }
-  return lastOfferMessage;
+  return lastMessageOfType;
 };
 
 function handleSignalingStateChangeEvent() {
@@ -493,7 +486,7 @@ function handleSignalingStateChangeEvent() {
 function handleConnectionStateChanged(pubkey: string) {
   window.log.info('handleConnectionStateChanged :', peerConnection?.connectionState);
 
-  if (peerConnection?.signalingState === 'closed') {
+  if (peerConnection?.signalingState === 'closed' || peerConnection?.connectionState === 'failed') {
     closeVideoCall();
   } else if (peerConnection?.connectionState === 'connected') {
     setIsRinging(false);
@@ -503,6 +496,7 @@ function handleConnectionStateChanged(pubkey: string) {
 
 function closeVideoCall() {
   window.log.info('closingVideoCall ');
+  setIsRinging(false);
   if (peerConnection) {
     peerConnection.ontrack = null;
     peerConnection.onicecandidate = null;
@@ -537,8 +531,15 @@ function closeVideoCall() {
   selectedCameraId = DEVICE_DISABLED_DEVICE_ID;
   selectedAudioInputId = DEVICE_DISABLED_DEVICE_ID;
   currentCallUUID = undefined;
-  callVideoListeners();
+
   window.inboxStore?.dispatch(setFullScreenCall(false));
+  remoteVideoStreamIsMuted = true;
+
+  makingOffer = false;
+  ignoreOffer = false;
+  isSettingRemoteAnswerPending = false;
+  lastOutgoingOfferTimestamp = -Infinity;
+  callVideoListeners();
 }
 
 function onDataChannelReceivedMessage(ev: MessageEvent<string>) {
@@ -558,7 +559,7 @@ function onDataChannelReceivedMessage(ev: MessageEvent<string>) {
       if (!foundEntry || !foundEntry.id) {
         return;
       }
-      handleCallTypeEndCall(foundEntry.id);
+      handleCallTypeEndCall(foundEntry.id, currentCallUUID);
 
       return;
     }
@@ -573,7 +574,7 @@ function onDataChannelReceivedMessage(ev: MessageEvent<string>) {
 }
 function onDataChannelOnOpen() {
   window.log.info('onDataChannelOnOpen: sending video status');
-
+  setIsRinging(false);
   sendVideoStatusViaDataChannel();
 }
 
@@ -593,8 +594,8 @@ function createOrGetPeerConnection(withPubkey: string, isAcceptingCall = false)
   dataChannel.onopen = onDataChannelOnOpen;
 
   if (!isAcceptingCall) {
-    peerConnection.onnegotiationneeded = async (event: Event) => {
-      await handleNegotiationNeededEvent(event, withPubkey);
+    peerConnection.onnegotiationneeded = async () => {
+      await handleNegotiationNeededEvent(withPubkey);
     };
   }
 
@@ -618,6 +619,24 @@ function createOrGetPeerConnection(withPubkey: string, isAcceptingCall = false)
     handleIceCandidates(event, withPubkey);
   };
 
+  peerConnection.oniceconnectionstatechange = () => {
+    window.log.info(
+      'oniceconnectionstatechange peerConnection.iceConnectionState: ',
+      peerConnection?.iceConnectionState
+    );
+
+    if (peerConnection && peerConnection?.iceConnectionState === 'disconnected') {
+      //this will trigger a negotation event with iceRestart set to true in the createOffer options set
+      global.setTimeout(() => {
+        window.log.info('onconnectionstatechange disconnected: restartIce()');
+
+        if (peerConnection?.iceConnectionState === 'disconnected') {
+          (peerConnection as any).restartIce();
+        }
+      }, 2000);
+    }
+  };
+
   return peerConnection;
 }
 
@@ -695,22 +714,31 @@ export async function USER_acceptIncomingCallRequest(fromSender: string) {
 // tslint:disable-next-line: function-name
 export async function USER_rejectIncomingCallRequest(fromSender: string) {
   setIsRinging(false);
-  const endCallMessage = new CallMessage({
-    type: SignalService.CallMessage.Type.END_CALL,
-    timestamp: Date.now(),
-    uuid: uuidv4(), // just send a random thing, we just want to reject the call
-  });
-  // delete all msg not from that uuid only but from that sender pubkey
+
+  const lastOfferMessage = findLastMessageTypeFromSender(
+    fromSender,
+    SignalService.CallMessage.Type.OFFER
+  );
+
+  const lastCallUUID = lastOfferMessage?.uuid;
+  window.log.info(`USER_rejectIncomingCallRequest ${ed25519Str(fromSender)}: ${lastCallUUID}`);
+  if (lastCallUUID) {
+    const endCallMessage = new CallMessage({
+      type: SignalService.CallMessage.Type.END_CALL,
+      timestamp: Date.now(),
+      uuid: lastCallUUID,
+    });
+    await getMessageQueue().sendToPubKeyNonDurably(PubKey.cast(fromSender), endCallMessage);
+
+    // delete all msg not from that uuid only but from that sender pubkey
+    clearCallCacheFromPubkeyAndUUID(fromSender, lastCallUUID);
+  }
 
   window.inboxStore?.dispatch(
     endCall({
       pubkey: fromSender,
     })
   );
-  window.log.info('USER_rejectIncomingCallRequest');
-  clearCallCacheFromPubkey(fromSender);
-
-  await getMessageQueue().sendToPubKeyNonDurably(PubKey.cast(fromSender), endCallMessage);
 
   const convos = getConversationController().getConversations();
   const callingConvos = convos.filter(convo => convo.callState !== undefined);
@@ -743,28 +771,18 @@ export async function USER_hangup(fromSender: string) {
 
   sendHangupViaDataChannel();
 
-  clearCallCacheFromPubkey(fromSender);
+  clearCallCacheFromPubkeyAndUUID(fromSender, currentCallUUID);
 
-  const convos = getConversationController().getConversations();
-  const callingConvos = convos.filter(convo => convo.callState !== undefined);
-  if (callingConvos.length > 0) {
-    // we just got a new offer from someone we are already in a call with
-    if (callingConvos.length === 1 && callingConvos[0].id === fromSender) {
-      closeVideoCall();
-    }
-  }
+  closeVideoCall();
 }
 
-export function handleCallTypeEndCall(sender: string) {
-  clearCallCacheFromPubkey(sender);
+export function handleCallTypeEndCall(sender: string, aboutCallUUID?: string) {
+  window.log.info('handling callMessage END_CALL:', aboutCallUUID);
 
-  window.log.info('handling callMessage END_CALL');
+  if (aboutCallUUID) {
+    clearCallCacheFromPubkeyAndUUID(sender, aboutCallUUID);
 
-  const convos = getConversationController().getConversations();
-  const callingConvos = convos.filter(convo => convo.callState !== undefined);
-  if (callingConvos.length > 0) {
-    // we just got a end call event from whoever we are in a call with
-    if (callingConvos.length === 1 && callingConvos[0].id === sender) {
+    if (aboutCallUUID === currentCallUUID) {
       closeVideoCall();
 
       window.inboxStore?.dispatch(endCall({ pubkey: sender }));
@@ -953,7 +971,9 @@ export async function handleCallTypeIceCandidates(
   window.log.info('handling callMessage ICE_CANDIDATES');
 
   pushCallMessageToCallCache(sender, remoteCallUUID, callMessage);
-  await addIceCandidateToExistingPeerConnection(callMessage);
+  if (currentCallUUID && callMessage.uuid === currentCallUUID) {
+    await addIceCandidateToExistingPeerConnection(callMessage);
+  }
 }
 
 async function addIceCandidateToExistingPeerConnection(callMessage: SignalService.CallMessage) {
@@ -987,8 +1007,8 @@ export async function handleOtherCallTypes(sender: string, callMessage: SignalSe
   pushCallMessageToCallCache(sender, remoteCallUUID, callMessage);
 }
 
-function clearCallCacheFromPubkey(sender: string) {
-  callCache.delete(sender);
+function clearCallCacheFromPubkeyAndUUID(sender: string, callUUID: string) {
+  callCache.get(sender)?.delete(callUUID);
 }
 
 function createCallCacheForPubkeyAndUUID(sender: string, uuid: string) {

ts/session/utils/RingingManager.ts

@@ -7,6 +7,7 @@ let ringingAudio: HTMLAudioElement | undefined;
 function stopRinging() {
   if (ringingAudio) {
     ringingAudio.pause();
+    ringingAudio.srcObject = null;
   }
 }
 
@@ -14,6 +15,7 @@ function startRinging() {
   if (!ringingAudio) {
     ringingAudio = new Audio(sound);
     ringingAudio.loop = true;
+    ringingAudio.volume = 0.6;
   }
   void ringingAudio.play();
 }

ts/state/ducks/conversations.ts

@@ -765,7 +765,7 @@ const conversationsSlice = createSlice({
     incomingCall(state: ConversationsStateType, action: PayloadAction<{ pubkey: string }>) {
       const callerPubkey = action.payload.pubkey;
       const existingCallState = state.conversationLookup[callerPubkey].callState;
-      if (existingCallState !== undefined && existingCallState !== 'none') {
+      if (existingCallState !== undefined) {
         return state;
       }
       const foundConvo = getConversationController().get(callerPubkey);
@@ -784,7 +784,7 @@ const conversationsSlice = createSlice({
     endCall(state: ConversationsStateType, action: PayloadAction<{ pubkey: string }>) {
       const callerPubkey = action.payload.pubkey;
       const existingCallState = state.conversationLookup[callerPubkey].callState;
-      if (!existingCallState || existingCallState === 'none') {
+      if (!existingCallState) {
         return state;
       }
 
@@ -796,7 +796,7 @@ const conversationsSlice = createSlice({
       // we have to update the model itself.
       // not the db (as we dont want to store that field in it)
       // and not the redux store directly as it gets overriden by the commit() of the conversationModel
-      foundConvo.callState = 'none';
+      foundConvo.callState = undefined;
 
       void foundConvo.commit();
       return state;
@@ -840,7 +840,7 @@ const conversationsSlice = createSlice({
     startingCallWith(state: ConversationsStateType, action: PayloadAction<{ pubkey: string }>) {
       const callerPubkey = action.payload.pubkey;
       const existingCallState = state.conversationLookup[callerPubkey].callState;
-      if (existingCallState && existingCallState !== 'none') {
+      if (existingCallState) {
         return state;
       }
       const foundConvo = getConversationController().get(callerPubkey);