Calling: Device Selection

This commit is contained in:
Josh Perez 2020-08-26 20:03:42 -04:00 committed by Josh Perez
parent 8b34294c97
commit 8ab1013f70
17 changed files with 1038 additions and 135 deletions

View File

@ -2725,5 +2725,25 @@
"example": "00:01"
}
}
},
"callingDeviceSelection__settings": {
"message": "Settings",
"description": "Title for device selection settings"
},
"callingDeviceSelection__label--video": {
"message": "Video",
"description": "Label for video input selector"
},
"callingDeviceSelection__label--audio-input": {
"message": "Microphone",
"description": "Label for audio input selector"
},
"callingDeviceSelection__label--audio-output": {
"message": "Speakers",
"description": "Label for audio output selector"
},
"callingDeviceSelection__select--no-device": {
"message": "No devices available",
"description": "Message for when there are no available devices to select for input/output audio or video"
}
}

View File

@ -129,7 +129,7 @@
"redux-ts-utils": "3.2.2",
"reselect": "4.0.0",
"rimraf": "2.6.2",
"ringrtc": "https://github.com/signalapp/signal-ringrtc-node.git#de96fa13f04e846bdcb0ae9be8e2dc80a932f2be",
"ringrtc": "https://github.com/signalapp/signal-ringrtc-node.git#0cd60f529d8f17734fe19e7195c9fd3c3f4271db",
"sanitize-filename": "1.6.3",
"sanitize.css": "11.0.0",
"semver": "5.4.1",

View File

@ -5984,6 +5984,21 @@ button.module-image__border-overlay:focus {
}
}
.module-ongoing-call__settings {
position: absolute;
top: 25px;
right: 25px;
&--button {
@include color-svg(
'../images/icons/v2/settings-solid-16.svg',
$color-white
);
height: 22px;
width: 22px;
}
}
// Module: Left Pane
.module-left-pane {
@ -8847,6 +8862,87 @@ button.module-image__border-overlay:focus {
margin-right: auto;
}
/* Calling: Device Selection */
.module-calling-device-selection {
position: relative;
}
.module-calling-device-selection__close-button {
@include button-reset;
@include light-theme {
@include color-svg('../images/x-shadow-16.svg', $color-gray-75);
}
@include dark-theme {
@include color-svg('../images/x-shadow-16.svg', $color-white);
}
height: 16px;
position: absolute;
right: 5px;
top: 0;
width: 16px;
z-index: 2;
@include keyboard-mode {
&:focus {
outline: 2px solid $ultramarine-ui-light;
}
}
}
.module-calling-device-selection__title {
@include font-title-2;
margin-top: 12px;
margin-bottom: 20px;
}
.module-calling-device-selection__label {
@include font-body-1-bold;
display: block;
margin-bottom: 16px;
}
.module-calling-device-selection__select {
margin-bottom: 20px;
position: relative;
select {
@include font-body-1;
-webkit-appearance: none;
border-radius: 4px;
border: 1px solid $color-gray-45;
cursor: pointer;
height: 40px;
outline: 0;
padding: 10px;
padding-right: 32px;
text-overflow: ellipsis;
width: 100%;
}
&::after {
border: 2px solid $color-gray-75;
border-radius: 2px;
border-right: 0;
border-top: 0;
content: ' ';
display: block;
height: 10px;
pointer-events: none;
position: absolute;
right: 15px;
top: 16px;
transform-origin: center;
transform: rotate(-45deg);
width: 10px;
z-index: 2;
}
}
/* Third-party module: react-tooltip-lite */
.react-tooltip-lite {

View File

@ -31,17 +31,18 @@ const defaultProps = {
callDetails,
callState: CallState.Accepted,
declineCall: action('decline-call'),
getVideoCapturer: () => ({}),
getVideoRenderer: () => ({}),
hangUp: action('hang-up'),
hasLocalAudio: true,
hasLocalVideo: true,
hasRemoteVideo: true,
i18n,
renderDeviceSelection: () => <div />,
setLocalAudio: action('set-local-audio'),
setLocalPreview: action('set-local-preview'),
setLocalVideo: action('set-local-video'),
setVideoCapturer: action('set-video-capturer'),
setVideoRenderer: action('set-video-renderer'),
setRendererCanvas: action('set-renderer-canvas'),
settingsDialogOpen: false,
toggleSettings: action('toggle-settings'),
};
const permutations = [

View File

@ -10,7 +10,10 @@ import { CallDetailsType } from '../state/ducks/calling';
type CallManagerPropsType = {
callDetails?: CallDetailsType;
callState?: CallState;
renderDeviceSelection: () => JSX.Element;
settingsDialogOpen: boolean;
};
type PropsType = IncomingCallBarPropsType &
CallScreenPropsType &
CallManagerPropsType;
@ -20,17 +23,18 @@ export const CallManager = ({
callDetails,
callState,
declineCall,
getVideoCapturer,
getVideoRenderer,
hangUp,
hasLocalAudio,
hasLocalVideo,
hasRemoteVideo,
i18n,
renderDeviceSelection,
setLocalAudio,
setLocalPreview,
setLocalVideo,
setVideoCapturer,
setVideoRenderer,
setRendererCanvas,
settingsDialogOpen,
toggleSettings,
}: PropsType): JSX.Element | null => {
if (!callDetails || !callState) {
return null;
@ -43,21 +47,23 @@ export const CallManager = ({
if (outgoing || ongoing) {
return (
<CallScreen
callDetails={callDetails}
callState={callState}
getVideoCapturer={getVideoCapturer}
getVideoRenderer={getVideoRenderer}
hangUp={hangUp}
hasLocalAudio={hasLocalAudio}
hasLocalVideo={hasLocalVideo}
i18n={i18n}
hasRemoteVideo={hasRemoteVideo}
setVideoCapturer={setVideoCapturer}
setVideoRenderer={setVideoRenderer}
setLocalAudio={setLocalAudio}
setLocalVideo={setLocalVideo}
/>
<>
<CallScreen
callDetails={callDetails}
callState={callState}
hangUp={hangUp}
hasLocalAudio={hasLocalAudio}
hasLocalVideo={hasLocalVideo}
i18n={i18n}
hasRemoteVideo={hasRemoteVideo}
setLocalPreview={setLocalPreview}
setRendererCanvas={setRendererCanvas}
setLocalAudio={setLocalAudio}
setLocalVideo={setLocalVideo}
toggleSettings={toggleSettings}
/>
{settingsDialogOpen && renderDeviceSelection()}
</>
);
}

View File

@ -30,17 +30,16 @@ const callDetails = {
const defaultProps = {
callDetails,
callState: CallState.Accepted,
getVideoCapturer: () => ({}),
getVideoRenderer: () => ({}),
hangUp: action('hang-up'),
hasLocalAudio: true,
hasLocalVideo: true,
hasRemoteVideo: true,
i18n,
setLocalAudio: action('set-local-audio'),
setLocalPreview: action('set-local-preview'),
setLocalVideo: action('set-local-video'),
setVideoCapturer: action('set-video-capturer'),
setVideoRenderer: action('set-video-renderer'),
setRendererCanvas: action('set-renderer-canvas'),
toggleSettings: action('toggle-settings'),
};
const permutations = [

View File

@ -4,14 +4,13 @@ import {
CallDetailsType,
HangUpType,
SetLocalAudioType,
SetLocalPreviewType,
SetLocalVideoType,
SetVideoCapturerType,
SetVideoRendererType,
SetRendererCanvasType,
} from '../state/ducks/calling';
import { Avatar } from './Avatar';
import { CallState } from '../types/Calling';
import { LocalizerType } from '../types/Util';
import { CanvasVideoRenderer, GumVideoCapturer } from '../window.d';
type CallingButtonProps = {
classNameSuffix: string;
@ -37,12 +36,6 @@ const CallingButton = ({
export type PropsType = {
callDetails?: CallDetailsType;
callState?: CallState;
getVideoCapturer: (
ref: React.RefObject<HTMLVideoElement>
) => GumVideoCapturer;
getVideoRenderer: (
ref: React.RefObject<HTMLCanvasElement>
) => CanvasVideoRenderer;
hangUp: (_: HangUpType) => void;
hasLocalAudio: boolean;
hasLocalVideo: boolean;
@ -50,8 +43,9 @@ export type PropsType = {
i18n: LocalizerType;
setLocalAudio: (_: SetLocalAudioType) => void;
setLocalVideo: (_: SetLocalVideoType) => void;
setVideoCapturer: (_: SetVideoCapturerType) => void;
setVideoRenderer: (_: SetVideoRendererType) => void;
setLocalPreview: (_: SetLocalPreviewType) => void;
setRendererCanvas: (_: SetRendererCanvasType) => void;
toggleSettings: () => void;
};
type StateType = {
@ -79,11 +73,6 @@ export class CallScreen extends React.Component<PropsType, StateType> {
this.controlsFadeTimer = null;
this.localVideoRef = React.createRef();
this.remoteVideoRef = React.createRef();
this.setVideoCapturerAndRenderer(
props.getVideoCapturer(this.localVideoRef),
props.getVideoRenderer(this.remoteVideoRef)
);
}
public componentDidMount() {
@ -92,6 +81,9 @@ export class CallScreen extends React.Component<PropsType, StateType> {
this.fadeControls();
document.addEventListener('keydown', this.handleKeyDown);
this.props.setLocalPreview({ element: this.localVideoRef });
this.props.setRendererCanvas({ element: this.remoteVideoRef });
}
public componentWillUnmount() {
@ -103,7 +95,8 @@ export class CallScreen extends React.Component<PropsType, StateType> {
if (this.controlsFadeTimer) {
clearTimeout(this.controlsFadeTimer);
}
this.setVideoCapturerAndRenderer(null, null);
this.props.setLocalPreview({ element: undefined });
this.props.setRendererCanvas({ element: undefined });
}
updateAcceptedTimer = () => {
@ -203,6 +196,8 @@ export class CallScreen extends React.Component<PropsType, StateType> {
hasLocalAudio,
hasLocalVideo,
hasRemoteVideo,
i18n,
toggleSettings,
} = this.props;
const { showControls } = this.state;
const isAudioOnly = !hasLocalVideo && !hasRemoteVideo;
@ -241,6 +236,13 @@ export class CallScreen extends React.Component<PropsType, StateType> {
{callDetails.title}
</div>
{this.renderMessage(callState)}
<div className="module-ongoing-call__settings">
<button
aria-label={i18n('callingDeviceSelection__settings')}
className="module-ongoing-call__settings--button"
onClick={toggleSettings}
/>
</div>
</div>
{hasRemoteVideo
? this.renderRemoteVideo()
@ -356,27 +358,4 @@ export class CallScreen extends React.Component<PropsType, StateType> {
}
return `${mins}:${secs}`;
}
private setVideoCapturerAndRenderer(
capturer: GumVideoCapturer | null,
renderer: CanvasVideoRenderer | null
) {
const { callDetails, setVideoCapturer, setVideoRenderer } = this.props;
if (!callDetails) {
return;
}
const { callId } = callDetails;
setVideoCapturer({
callId,
capturer,
});
setVideoRenderer({
callId,
renderer,
});
}
}

View File

@ -0,0 +1,145 @@
import * as React from 'react';
import { CallingDeviceSelection, Props } from './CallingDeviceSelection';
// @ts-ignore
import { setup as setupI18n } from '../../js/modules/i18n';
// @ts-ignore
import enMessages from '../../_locales/en/messages.json';
import { action } from '@storybook/addon-actions';
import { storiesOf } from '@storybook/react';
const i18n = setupI18n('en', enMessages);
const audioDevice = {
name: '',
index: 0,
same_name_index: 0,
};
const createProps = ({
availableMicrophones = [],
availableSpeakers = [],
selectedMicrophone = audioDevice,
selectedSpeaker = audioDevice,
availableCameras = [],
selectedCamera = '',
}: Partial<Props> = {}): Props => ({
availableCameras,
availableMicrophones,
availableSpeakers,
changeIODevice: action('change-io-device'),
i18n,
selectedCamera,
selectedMicrophone,
selectedSpeaker,
toggleSettings: action('toggle-settings'),
});
const stories = storiesOf('Components/CallingDeviceSelection', module);
stories.add('Default', () => {
return <CallingDeviceSelection {...createProps()} />;
});
stories.add('Some Devices', () => {
const availableSpeakers = [
{
name: 'Default - Internal Microphone',
index: 0,
same_name_index: 0,
},
{
name: "Natalie's Airpods (Bluetooth)",
index: 1,
same_name_index: 1,
},
{
name: 'UE Boom (Bluetooth)',
index: 2,
same_name_index: 2,
},
];
const selectedSpeaker = availableSpeakers[0];
const props = createProps({
availableSpeakers,
selectedSpeaker,
});
return <CallingDeviceSelection {...props} />;
});
stories.add('All Devices', () => {
const availableSpeakers = [
{
name: 'Default - Internal Speakers',
index: 0,
same_name_index: 0,
},
{
name: "Natalie's Airpods (Bluetooth)",
index: 1,
same_name_index: 1,
},
{
name: 'UE Boom (Bluetooth)',
index: 2,
same_name_index: 2,
},
];
const selectedSpeaker = availableSpeakers[0];
const availableMicrophones = [
{
name: 'Default - Internal Microphone',
index: 0,
same_name_index: 0,
},
{
name: "Natalie's Airpods (Bluetooth)",
index: 1,
same_name_index: 1,
},
];
const selectedMicrophone = availableMicrophones[0];
const availableCameras = [
{
deviceId:
'dfbe6effe70b0611ba0fdc2a9ea3f39f6cb110e6687948f7e5f016c111b7329c',
groupId:
'63ee218d2446869e40adfc958ff98263e51f74382b0143328ee4826f20a76f47',
kind: 'videoinput' as MediaDeviceKind,
label: 'FaceTime HD Camera (Built-in) (9fba:bced)',
toJSON() {
return '';
},
},
{
deviceId:
'e2db196a31d50ff9b135299dc0beea67f65b1a25a06d8a4ce76976751bb7a08d',
groupId:
'218ba7f00d7b1239cca15b9116769e5e7d30cc01104ebf84d667643661e0ecf9',
kind: 'videoinput' as MediaDeviceKind,
label: 'Logitech Webcam (4e72:9058)',
toJSON() {
return '';
},
},
];
const selectedCamera =
'dfbe6effe70b0611ba0fdc2a9ea3f39f6cb110e6687948f7e5f016c111b7329c';
const props = createProps({
availableCameras,
availableMicrophones,
availableSpeakers,
selectedCamera,
selectedMicrophone,
selectedSpeaker,
});
return <CallingDeviceSelection {...props} />;
});

View File

@ -0,0 +1,192 @@
import * as React from 'react';
import { ConfirmationModal } from './ConfirmationModal';
import { LocalizerType } from '../types/Util';
import {
AudioDevice,
CallingDeviceType,
ChangeIODevicePayloadType,
MediaDeviceSettings,
} from '../types/Calling';
export type Props = MediaDeviceSettings & {
changeIODevice: (payload: ChangeIODevicePayloadType) => void;
i18n: LocalizerType;
toggleSettings: () => void;
};
function renderAudioOptions(
devices: Array<AudioDevice>,
i18n: LocalizerType,
selectedDevice: AudioDevice | undefined
): JSX.Element {
if (!devices.length) {
return (
<option aria-selected={true}>
{i18n('callingDeviceSelection__select--no-device')}
</option>
);
}
return (
<>
{devices.map((device: AudioDevice) => {
const isSelected =
selectedDevice && selectedDevice.index === device.index;
return (
<option
aria-selected={isSelected}
key={device.index}
value={device.index}
>
{device.name}
</option>
);
})}
</>
);
}
function renderVideoOptions(
devices: Array<MediaDeviceInfo>,
i18n: LocalizerType,
selectedCamera: string | undefined
): JSX.Element {
if (!devices.length) {
return (
<option aria-selected={true}>
{i18n('callingDeviceSelection__select--no-device')}
</option>
);
}
return (
<>
{devices.map((device: MediaDeviceInfo) => {
const isSelected = selectedCamera === device.deviceId;
return (
<option
aria-selected={isSelected}
key={device.deviceId}
value={device.deviceId}
>
{device.label}
</option>
);
})}
</>
);
}
function createAudioChangeHandler(
devices: Array<AudioDevice>,
changeIODevice: (payload: ChangeIODevicePayloadType) => void,
type: CallingDeviceType.SPEAKER | CallingDeviceType.MICROPHONE
) {
return (ev: React.FormEvent<HTMLSelectElement>): void => {
changeIODevice({
type,
selectedDevice: devices[Number(ev.currentTarget.value)],
});
};
}
function createCameraChangeHandler(
changeIODevice: (payload: ChangeIODevicePayloadType) => void
) {
return (ev: React.FormEvent<HTMLSelectElement>): void => {
changeIODevice({
type: CallingDeviceType.CAMERA,
selectedDevice: String(ev.currentTarget.value),
});
};
}
export const CallingDeviceSelection = ({
availableCameras,
availableMicrophones,
availableSpeakers,
changeIODevice,
i18n,
selectedCamera,
selectedMicrophone,
selectedSpeaker,
toggleSettings,
}: Props): JSX.Element => {
const selectedMicrophoneIndex = selectedMicrophone
? selectedMicrophone.index
: undefined;
const selectedSpeakerIndex = selectedSpeaker
? selectedSpeaker.index
: undefined;
return (
<ConfirmationModal actions={[]} i18n={i18n} onClose={toggleSettings}>
<div className="module-calling-device-selection">
<button
className="module-calling-device-selection__close-button"
onClick={toggleSettings}
tabIndex={0}
/>
</div>
<h1 className="module-calling-device-selection__title">
{i18n('callingDeviceSelection__settings')}
</h1>
<label className="module-calling-device-selection__label">
{i18n('callingDeviceSelection__label--video')}
</label>
<div className="module-calling-device-selection__select">
<select
disabled={!availableCameras.length}
name="video"
// tslint:disable-next-line react-a11y-no-onchange
onChange={createCameraChangeHandler(changeIODevice)}
value={selectedCamera}
>
{renderVideoOptions(availableCameras, i18n, selectedCamera)}
</select>
</div>
<label className="module-calling-device-selection__label">
{i18n('callingDeviceSelection__label--audio-input')}
</label>
<div className="module-calling-device-selection__select">
<select
disabled={!availableMicrophones.length}
name="audio-input"
// tslint:disable-next-line react-a11y-no-onchange
onChange={createAudioChangeHandler(
availableMicrophones,
changeIODevice,
CallingDeviceType.MICROPHONE
)}
value={selectedMicrophoneIndex}
>
{renderAudioOptions(availableMicrophones, i18n, selectedMicrophone)}
</select>
</div>
<label className="module-calling-device-selection__label">
{i18n('callingDeviceSelection__label--audio-output')}
</label>
<div className="module-calling-device-selection__select">
<select
disabled={!availableSpeakers.length}
name="audio-output"
// tslint:disable-next-line react-a11y-no-onchange
onChange={createAudioChangeHandler(
availableSpeakers,
changeIODevice,
CallingDeviceType.SPEAKER
)}
value={selectedSpeakerIndex}
>
{renderAudioOptions(availableSpeakers, i18n, selectedSpeaker)}
</select>
</div>
</ConfirmationModal>
);
};

View File

@ -5,12 +5,13 @@ import {
CallLogLevel,
CallSettings,
CallState,
CanvasVideoRenderer,
DeviceId,
GumVideoCapturer,
RingRTC,
UserId,
VideoCapturer,
VideoRenderer,
} from 'ringrtc';
import {
ActionsType as UxActionsType,
CallDetailsType,
@ -18,6 +19,7 @@ import {
import { CallingMessageClass, EnvelopeClass } from '../textsecure.d';
import { ConversationModelType } from '../model-types.d';
import is from '@sindresorhus/is';
import { AudioDevice, MediaDeviceSettings } from '../types/Calling';
export {
CallState,
@ -36,7 +38,16 @@ export type CallHistoryDetailsType = {
};
export class CallingClass {
readonly videoCapturer: GumVideoCapturer;
readonly videoRenderer: CanvasVideoRenderer;
private uxActions?: UxActionsType;
private lastMediaDeviceSettings?: MediaDeviceSettings;
private deviceReselectionTimer?: NodeJS.Timeout;
constructor() {
this.videoCapturer = new GumVideoCapturer(640, 480, 30);
this.videoRenderer = new CanvasVideoRenderer();
}
initialize(uxActions: UxActionsType): void {
this.uxActions = uxActions;
@ -80,11 +91,6 @@ export class CallingClass {
return;
}
if (RingRTC.call && RingRTC.call.state !== CallState.Ended) {
window.log.info('Call already in progress, new call not allowed.');
return;
}
const remoteUserId = this.getRemoteUserIdFromConversation(conversation);
if (!remoteUserId || !this.localDeviceId) {
window.log.error('Missing identifier, new call not allowed.');
@ -97,15 +103,26 @@ export class CallingClass {
return;
}
const callSettings = await this.getCallSettings(conversation);
// Check state after awaiting to debounce call button.
if (RingRTC.call && RingRTC.call.state !== CallState.Ended) {
window.log.info('Call already in progress, new call not allowed.');
return;
}
// We could make this faster by getting the call object
// from the RingRTC before we lookup the ICE servers.
const call = RingRTC.startOutgoingCall(
remoteUserId,
isVideoCall,
this.localDeviceId,
await this.getCallSettings(conversation)
callSettings
);
await this.startDeviceReselectionTimer();
RingRTC.setVideoCapturer(call.callId, this.videoCapturer);
RingRTC.setVideoRenderer(call.callId, this.videoRenderer);
this.attachToCall(conversation, call);
this.uxActions.outgoingCall({
@ -116,6 +133,9 @@ export class CallingClass {
async accept(callId: CallId, asVideoCall: boolean) {
const haveMediaPermissions = await this.requestPermissions(asVideoCall);
if (haveMediaPermissions) {
await this.startDeviceReselectionTimer();
RingRTC.setVideoCapturer(callId, this.videoCapturer);
RingRTC.setVideoRenderer(callId, this.videoRenderer);
RingRTC.accept(callId, asVideoCall);
} else {
window.log.info('Permissions were denied, call not allowed, hanging up.');
@ -139,12 +159,230 @@ export class CallingClass {
RingRTC.setOutgoingVideo(callId, enabled);
}
setVideoCapturer(callId: CallId, capturer: VideoCapturer | null) {
RingRTC.setVideoCapturer(callId, capturer);
private async startDeviceReselectionTimer(): Promise<void> {
// Poll once
await this.pollForMediaDevices();
// Start the timer
if (!this.deviceReselectionTimer) {
this.deviceReselectionTimer = setInterval(async () => {
await this.pollForMediaDevices();
}, 3000);
}
}
setVideoRenderer(callId: CallId, renderer: VideoRenderer | null) {
RingRTC.setVideoRenderer(callId, renderer);
private stopDeviceReselectionTimer() {
if (this.deviceReselectionTimer) {
clearInterval(this.deviceReselectionTimer);
this.deviceReselectionTimer = undefined;
}
}
// tslint:disable-next-line cyclomatic-complexity
private mediaDeviceSettingsEqual(
a?: MediaDeviceSettings,
b?: MediaDeviceSettings
): boolean {
if (!a && !b) {
return true;
}
if (!a || !b) {
return false;
}
if (
a.availableCameras.length !== b.availableCameras.length ||
a.availableMicrophones.length !== b.availableMicrophones.length ||
a.availableSpeakers.length !== b.availableSpeakers.length
) {
return false;
}
for (let i = 0; i < a.availableCameras.length; i++) {
if (
a.availableCameras[i].deviceId !== b.availableCameras[i].deviceId ||
a.availableCameras[i].groupId !== b.availableCameras[i].groupId ||
a.availableCameras[i].label !== b.availableCameras[i].label
) {
return false;
}
}
for (let i = 0; i < a.availableMicrophones.length; i++) {
if (
a.availableMicrophones[i].name !== b.availableMicrophones[i].name ||
a.availableMicrophones[i].unique_id !==
b.availableMicrophones[i].unique_id ||
a.availableMicrophones[i].same_name_index !==
b.availableMicrophones[i].same_name_index
) {
return false;
}
}
for (let i = 0; i < a.availableSpeakers.length; i++) {
if (
a.availableSpeakers[i].name !== b.availableSpeakers[i].name ||
a.availableSpeakers[i].unique_id !== b.availableSpeakers[i].unique_id ||
a.availableSpeakers[i].same_name_index !==
b.availableSpeakers[i].same_name_index
) {
return false;
}
}
if (
(a.selectedCamera && !b.selectedCamera) ||
(!a.selectedCamera && b.selectedCamera) ||
(a.selectedMicrophone && !b.selectedMicrophone) ||
(!a.selectedMicrophone && b.selectedMicrophone) ||
(a.selectedSpeaker && !b.selectedSpeaker) ||
(!a.selectedSpeaker && b.selectedSpeaker)
) {
return false;
}
if (
a.selectedCamera &&
b.selectedCamera &&
a.selectedCamera !== b.selectedCamera
) {
return false;
}
if (
a.selectedMicrophone &&
b.selectedMicrophone &&
a.selectedMicrophone.index !== b.selectedMicrophone.index
) {
return false;
}
if (
a.selectedSpeaker &&
b.selectedSpeaker &&
a.selectedSpeaker.index !== b.selectedSpeaker.index
) {
return false;
}
return true;
}
private async pollForMediaDevices(): Promise<void> {
const newSettings = await this.getMediaDeviceSettings();
if (
!this.mediaDeviceSettingsEqual(this.lastMediaDeviceSettings, newSettings)
) {
window.log.info(
'MediaDevice: available devices changed (from->to)',
this.lastMediaDeviceSettings,
newSettings
);
await this.selectPreferredMediaDevices(newSettings);
this.lastMediaDeviceSettings = newSettings;
this.uxActions?.refreshIODevices(newSettings);
}
}
async getMediaDeviceSettings(): Promise<MediaDeviceSettings> {
const availableMicrophones = RingRTC.getAudioInputs();
const preferredMicrophone = window.storage.get(
'preferred-audio-input-device'
);
const selectedMicIndex = this.findBestMatchingDeviceIndex(
availableMicrophones,
preferredMicrophone
);
const selectedMicrophone =
selectedMicIndex !== undefined
? availableMicrophones[selectedMicIndex]
: undefined;
const availableSpeakers = RingRTC.getAudioOutputs();
const preferredSpeaker = window.storage.get(
'preferred-audio-output-device'
);
const selectedSpeakerIndex = this.findBestMatchingDeviceIndex(
availableSpeakers,
preferredSpeaker
);
const selectedSpeaker =
selectedSpeakerIndex !== undefined
? availableSpeakers[selectedSpeakerIndex]
: undefined;
const availableCameras = await window.Signal.Services.calling.videoCapturer.enumerateDevices();
const preferredCamera = window.storage.get('preferred-video-input-device');
const selectedCamera = this.findBestMatchingCamera(
availableCameras,
preferredCamera
);
return {
availableMicrophones,
availableSpeakers,
selectedMicrophone,
selectedSpeaker,
availableCameras,
selectedCamera,
};
}
findBestMatchingDeviceIndex(
available: Array<AudioDevice>,
preferred: AudioDevice | undefined
): number | undefined {
if (!preferred) {
// No preference stored
return undefined;
}
// Match by UUID first, if available
if (preferred.unique_id) {
const matchIndex = available.findIndex(
d => d.unique_id === preferred.unique_id
);
if (matchIndex !== -1) {
return matchIndex;
}
}
// Match by name second, and if there are multiple such names - by instance index.
const matchingNames = available.filter(d => d.name === preferred.name);
if (matchingNames.length > preferred.same_name_index) {
return matchingNames[preferred.same_name_index].index;
}
if (matchingNames.length > 0) {
return matchingNames[0].index;
}
// Nothing matches; take the first device if there are any
return available.length > 0 ? 0 : undefined;
}
findBestMatchingCamera(
available: Array<MediaDeviceInfo>,
preferred?: string
): string | undefined {
const matchingId = available.filter(d => d.deviceId === preferred);
const nonInfrared = available.filter(d => !d.label.includes('IR Camera'));
/// By default, pick the first non-IR camera (but allow the user to pick the infrared if they so desire)
if (matchingId.length > 0) {
return matchingId[0].deviceId;
} else if (nonInfrared.length > 0) {
return nonInfrared[0].deviceId;
} else {
return undefined;
}
}
setPreferredMicrophone(device: AudioDevice) {
window.log.info('MediaDevice: setPreferredMicrophone', device);
window.storage.put('preferred-audio-input-device', device);
RingRTC.setAudioInput(device.index);
}
setPreferredSpeaker(device: AudioDevice) {
window.log.info('MediaDevice: setPreferredSpeaker', device);
window.storage.put('preferred-audio-output-device', device);
RingRTC.setAudioOutput(device.index);
}
async setPreferredCamera(device: string) {
window.log.info('MediaDevice: setPreferredCamera', device);
window.storage.put('preferred-video-input-device', device);
await this.videoCapturer.setPreferredDevice(device);
}
async handleCallingMessage(
@ -176,6 +414,30 @@ export class CallingClass {
);
}
private async selectPreferredMediaDevices(
settings: MediaDeviceSettings
): Promise<void> {
// Assume that the MediaDeviceSettings have been obtained very recently and the index is still valid (no devices have been plugged in in between).
if (settings.selectedMicrophone) {
window.log.info(
'MediaDevice: selecting microphone',
settings.selectedMicrophone
);
RingRTC.setAudioInput(settings.selectedMicrophone.index);
}
if (settings.selectedSpeaker) {
window.log.info(
'MediaDevice: selecting speaker',
settings.selectedMicrophone
);
RingRTC.setAudioOutput(settings.selectedSpeaker.index);
}
if (settings.selectedCamera) {
window.log.info('MediaDevice: selecting camera', settings.selectedCamera);
await this.videoCapturer.setPreferredDevice(settings.selectedCamera);
}
}
private async requestCameraPermissions(): Promise<boolean> {
const cameraPermission = await window.getMediaCameraPermissions();
if (!cameraPermission) {
@ -325,6 +587,8 @@ export class CallingClass {
acceptedTime = Date.now();
} else if (call.state === CallState.Ended) {
this.addCallHistoryForEndedCall(conversation, call, acceptedTime);
this.stopDeviceReselectionTimer();
this.lastMediaDeviceSettings = undefined;
}
uxActions.callStateChange({
callState: call.state,

View File

@ -1,7 +1,11 @@
import { notify } from '../../services/notify';
import { calling, VideoCapturer, VideoRenderer } from '../../services/calling';
import { CallState } from '../../types/Calling';
import { CanvasVideoRenderer, GumVideoCapturer } from '../../window.d';
import { calling } from '../../services/calling';
import {
CallingDeviceType,
CallState,
ChangeIODevicePayloadType,
MediaDeviceSettings,
} from '../../types/Calling';
import { ColorType } from '../../types/Colors';
import { NoopActionType } from './noop';
import { callingTones } from '../../util/callingTones';
@ -28,12 +32,13 @@ export type CallDetailsType = {
title: string;
};
export type CallingStateType = {
export type CallingStateType = MediaDeviceSettings & {
callDetails?: CallDetailsType;
callState?: CallState;
hasLocalAudio: boolean;
hasLocalVideo: boolean;
hasRemoteVideo: boolean;
settingsDialogOpen: boolean;
};
export type AcceptCallType = {
@ -76,14 +81,12 @@ export type SetLocalVideoType = {
enabled: boolean;
};
export type SetVideoCapturerType = {
callId: CallId;
capturer: CanvasVideoRenderer | null;
export type SetLocalPreviewType = {
element: React.RefObject<HTMLVideoElement> | undefined;
};
export type SetVideoRendererType = {
callId: CallId;
renderer: GumVideoCapturer | null;
export type SetRendererCanvasType = {
element: React.RefObject<HTMLCanvasElement> | undefined;
};
// Actions
@ -91,14 +94,18 @@ export type SetVideoRendererType = {
const ACCEPT_CALL = 'calling/ACCEPT_CALL';
const CALL_STATE_CHANGE = 'calling/CALL_STATE_CHANGE';
const CALL_STATE_CHANGE_FULFILLED = 'calling/CALL_STATE_CHANGE_FULFILLED';
const CHANGE_IO_DEVICE = 'calling/CHANGE_IO_DEVICE';
const CHANGE_IO_DEVICE_FULFILLED = 'calling/CHANGE_IO_DEVICE_FULFILLED';
const DECLINE_CALL = 'calling/DECLINE_CALL';
const HANG_UP = 'calling/HANG_UP';
const INCOMING_CALL = 'calling/INCOMING_CALL';
const OUTGOING_CALL = 'calling/OUTGOING_CALL';
const REFRESH_IO_DEVICES = 'calling/REFRESH_IO_DEVICES';
const REMOTE_VIDEO_CHANGE = 'calling/REMOTE_VIDEO_CHANGE';
const SET_LOCAL_AUDIO = 'calling/SET_LOCAL_AUDIO';
const SET_LOCAL_VIDEO = 'calling/SET_LOCAL_VIDEO';
const SET_LOCAL_VIDEO_FULFILLED = 'calling/SET_LOCAL_VIDEO_FULFILLED';
const TOGGLE_SETTINGS = 'calling/TOGGLE_SETTINGS';
type AcceptCallActionType = {
type: 'calling/ACCEPT_CALL';
@ -115,6 +122,16 @@ type CallStateChangeFulfilledActionType = {
payload: CallStateChangeType;
};
type ChangeIODeviceActionType = {
type: 'calling/CHANGE_IO_DEVICE';
payload: Promise<ChangeIODevicePayloadType>;
};
type ChangeIODeviceFulfilledActionType = {
type: 'calling/CHANGE_IO_DEVICE_FULFILLED';
payload: ChangeIODevicePayloadType;
};
type DeclineCallActionType = {
type: 'calling/DECLINE_CALL';
payload: DeclineCallType;
@ -135,6 +152,11 @@ type OutgoingCallActionType = {
payload: OutgoingCallType;
};
type RefreshIODevicesActionType = {
type: 'calling/REFRESH_IO_DEVICES';
payload: MediaDeviceSettings;
};
type RemoteVideoChangeActionType = {
type: 'calling/REMOTE_VIDEO_CHANGE';
payload: RemoteVideoChangeType;
@ -155,18 +177,26 @@ type SetLocalVideoFulfilledActionType = {
payload: SetLocalVideoType;
};
type ToggleSettingsActionType = {
type: 'calling/TOGGLE_SETTINGS';
};
export type CallingActionType =
| AcceptCallActionType
| CallStateChangeActionType
| CallStateChangeFulfilledActionType
| ChangeIODeviceActionType
| ChangeIODeviceFulfilledActionType
| DeclineCallActionType
| HangUpActionType
| IncomingCallActionType
| OutgoingCallActionType
| RefreshIODevicesActionType
| RemoteVideoChangeActionType
| SetLocalAudioActionType
| SetLocalVideoActionType
| SetLocalVideoFulfilledActionType;
| SetLocalVideoFulfilledActionType
| ToggleSettingsActionType;
// Action Creators
@ -197,6 +227,29 @@ function callStateChange(
};
}
function changeIODevice(
payload: ChangeIODevicePayloadType
): ChangeIODeviceActionType {
return {
type: CHANGE_IO_DEVICE,
payload: doChangeIODevice(payload),
};
}
async function doChangeIODevice(
payload: ChangeIODevicePayloadType
): Promise<ChangeIODevicePayloadType> {
if (payload.type === CallingDeviceType.CAMERA) {
await calling.setPreferredCamera(payload.selectedDevice);
} else if (payload.type === CallingDeviceType.MICROPHONE) {
calling.setPreferredMicrophone(payload.selectedDevice);
} else if (payload.type === CallingDeviceType.SPEAKER) {
calling.setPreferredSpeaker(payload.selectedDevice);
}
return payload;
}
async function doCallStateChange(
payload: CallStateChangeType
): Promise<CallStateChangeType> {
@ -208,12 +261,11 @@ async function doCallStateChange(
bounceAppIconStart();
}
if (callState !== CallState.Ringing) {
callingTones.stopRingtone();
await callingTones.stopRingtone();
bounceAppIconStop();
}
if (callState === CallState.Ended) {
// tslint:disable-next-line no-floating-promises
callingTones.playEndCall();
await callingTones.playEndCall();
}
return payload;
}
@ -275,6 +327,15 @@ function outgoingCall(payload: OutgoingCallType): OutgoingCallActionType {
};
}
function refreshIODevices(
payload: MediaDeviceSettings
): RefreshIODevicesActionType {
return {
type: REFRESH_IO_DEVICES,
payload,
};
}
function remoteVideoChange(
payload: RemoteVideoChangeType
): RemoteVideoChangeActionType {
@ -284,8 +345,8 @@ function remoteVideoChange(
};
}
function setVideoCapturer(payload: SetVideoCapturerType): NoopActionType {
calling.setVideoCapturer(payload.callId, payload.capturer as VideoCapturer);
function setLocalPreview(payload: SetLocalPreviewType): NoopActionType {
calling.videoCapturer.setLocalPreview(payload.element);
return {
type: 'NOOP',
@ -293,8 +354,8 @@ function setVideoCapturer(payload: SetVideoCapturerType): NoopActionType {
};
}
function setVideoRenderer(payload: SetVideoRendererType): NoopActionType {
calling.setVideoRenderer(payload.callId, payload.renderer as VideoRenderer);
function setRendererCanvas(payload: SetRendererCanvasType): NoopActionType {
calling.videoRenderer.setCanvas(payload.element);
return {
type: 'NOOP',
@ -318,6 +379,12 @@ function setLocalVideo(payload: SetLocalVideoType): SetLocalVideoActionType {
};
}
function toggleSettings(): ToggleSettingsActionType {
return {
type: TOGGLE_SETTINGS,
};
}
async function doSetLocalVideo(
payload: SetLocalVideoType
): Promise<SetLocalVideoType> {
@ -335,15 +402,18 @@ async function doSetLocalVideo(
export const actions = {
acceptCall,
callStateChange,
changeIODevice,
declineCall,
hangUp,
incomingCall,
outgoingCall,
refreshIODevices,
remoteVideoChange,
setVideoCapturer,
setVideoRenderer,
setLocalPreview,
setRendererCanvas,
setLocalAudio,
setLocalVideo,
toggleSettings,
};
export type ActionsType = typeof actions;
@ -352,14 +422,22 @@ export type ActionsType = typeof actions;
function getEmptyState(): CallingStateType {
return {
availableCameras: [],
availableMicrophones: [],
availableSpeakers: [],
callDetails: undefined,
callState: undefined,
hasLocalAudio: false,
hasLocalVideo: false,
hasRemoteVideo: false,
selectedCamera: undefined,
selectedMicrophone: undefined,
selectedSpeaker: undefined,
settingsDialogOpen: false,
};
}
// tslint:disable-next-line max-func-body-length
export function reducer(
state: CallingStateType = getEmptyState(),
action: CallingActionType
@ -425,5 +503,51 @@ export function reducer(
};
}
if (action.type === CHANGE_IO_DEVICE_FULFILLED) {
const { selectedDevice } = action.payload;
const nextState = Object.create(null);
if (action.payload.type === CallingDeviceType.CAMERA) {
nextState.selectedCamera = selectedDevice;
} else if (action.payload.type === CallingDeviceType.MICROPHONE) {
nextState.selectedMicrophone = selectedDevice;
} else if (action.payload.type === CallingDeviceType.SPEAKER) {
nextState.selectedSpeaker = selectedDevice;
}
return {
...state,
...nextState,
};
}
if (action.type === REFRESH_IO_DEVICES) {
const {
availableMicrophones,
selectedMicrophone,
availableSpeakers,
selectedSpeaker,
availableCameras,
selectedCamera,
} = action.payload;
return {
...state,
availableMicrophones,
selectedMicrophone,
availableSpeakers,
selectedSpeaker,
availableCameras,
selectedCamera,
};
}
if (action.type === TOGGLE_SETTINGS) {
return {
...state,
settingsDialogOpen: !state.settingsDialogOpen,
};
}
return state;
}

View File

@ -1,20 +1,26 @@
import { RefObject } from 'react';
import React from 'react';
import { connect } from 'react-redux';
import { CanvasVideoRenderer, GumVideoCapturer } from 'ringrtc';
import { mapDispatchToProps } from '../actions';
import { CallManager } from '../../components/CallManager';
import { StateType } from '../reducer';
import { getIntl } from '../selectors/user';
import { SmartCallingDeviceSelection } from './CallingDeviceSelection';
// Workaround: A react component's required properties are filtering up through connect()
// https://github.com/DefinitelyTyped/DefinitelyTyped/issues/31363
const FilteredCallingDeviceSelection = SmartCallingDeviceSelection as any;
function renderDeviceSelection(): JSX.Element {
return <FilteredCallingDeviceSelection />;
}
const mapStateToProps = (state: StateType) => {
return {
...state.calling,
i18n: getIntl(state),
getVideoCapturer: (localVideoRef: RefObject<HTMLVideoElement>) =>
new GumVideoCapturer(640, 480, 30, localVideoRef),
getVideoRenderer: (remoteVideoRef: RefObject<HTMLCanvasElement>) =>
new CanvasVideoRenderer(remoteVideoRef),
renderDeviceSelection,
};
};

View File

@ -0,0 +1,31 @@
import { connect } from 'react-redux';
import { mapDispatchToProps } from '../actions';
import { CallingDeviceSelection } from '../../components/CallingDeviceSelection';
import { StateType } from '../reducer';
import { getIntl } from '../selectors/user';
const mapStateToProps = (state: StateType) => {
const {
availableMicrophones,
availableSpeakers,
selectedMicrophone,
selectedSpeaker,
availableCameras,
selectedCamera,
} = state.calling;
return {
availableCameras,
availableMicrophones,
availableSpeakers,
i18n: getIntl(state),
selectedCamera,
selectedMicrophone,
selectedSpeaker,
};
};
const smart = connect(mapStateToProps, mapDispatchToProps);
export const SmartCallingDeviceSelection = smart(CallingDeviceSelection);

View File

@ -1,3 +1,15 @@
// Must be kept in sync with RingRTC.AudioDevice
export interface AudioDevice {
// Name, present on every platform.
name: string;
// Index of this device, starting from 0.
index: number;
// Index of this device out of all devices sharing the same name.
same_name_index: number;
// If present, a unique and stable identifier of this device. Only available on WIndows.
unique_id?: string;
}
// This must be kept in sync with RingRTC.CallState.
export enum CallState {
Prering = 'init',
@ -6,3 +18,23 @@ export enum CallState {
Reconnecting = 'connecting',
Ended = 'ended',
}
export enum CallingDeviceType {
CAMERA,
MICROPHONE,
SPEAKER,
}
export type MediaDeviceSettings = {
availableMicrophones: Array<AudioDevice>;
selectedMicrophone: AudioDevice | undefined;
availableSpeakers: Array<AudioDevice>;
selectedSpeaker: AudioDevice | undefined;
availableCameras: Array<MediaDeviceInfo>;
selectedCamera: string | undefined;
};
export type ChangeIODevicePayloadType =
| { type: CallingDeviceType.CAMERA; selectedDevice: string }
| { type: CallingDeviceType.MICROPHONE; selectedDevice: AudioDevice }
| { type: CallingDeviceType.SPEAKER; selectedDevice: AudioDevice };

View File

@ -1,43 +1,51 @@
import { Sound, SoundOpts } from './Sound';
import { Sound } from './Sound';
import PQueue from 'p-queue';
async function playSound(howlProps: SoundOpts): Promise<Sound | undefined> {
const canPlayTone = await window.getCallRingtoneNotification();
if (!canPlayTone) {
return;
}
const tone = new Sound(howlProps);
await tone.play();
return tone;
}
const ringtoneEventQueue = new PQueue({ concurrency: 1 });
class CallingTones {
private ringtone?: Sound;
async playEndCall() {
await playSound({
const canPlayTone = await window.getCallRingtoneNotification();
if (!canPlayTone) {
return;
}
const tone = new Sound({
src: 'sounds/navigation-cancel.ogg',
});
await tone.play();
}
async playRingtone() {
if (this.ringtone) {
this.stopRingtone();
}
await ringtoneEventQueue.add(async () => {
if (this.ringtone) {
this.ringtone.stop();
this.ringtone = undefined;
}
this.ringtone = await playSound({
loop: true,
src: 'sounds/ringtone_minimal.ogg',
const canPlayTone = await window.getCallRingtoneNotification();
if (!canPlayTone) {
return;
}
this.ringtone = new Sound({
loop: true,
src: 'sounds/ringtone_minimal.ogg',
});
await this.ringtone.play();
});
}
stopRingtone() {
if (this.ringtone) {
this.ringtone.stop();
this.ringtone = undefined;
}
async stopRingtone() {
await ringtoneEventQueue.add(async () => {
if (this.ringtone) {
this.ringtone.stop();
this.ringtone = undefined;
}
});
}
}

View File

@ -11304,7 +11304,7 @@
"rule": "React-createRef",
"path": "ts/components/CallScreen.tsx",
"line": " this.localVideoRef = React.createRef();",
"lineNumber": 80,
"lineNumber": 74,
"reasonCategory": "usageTrusted",
"updated": "2020-06-02T21:51:34.813Z",
"reasonDetail": "Used to render local preview video"

View File

@ -14923,9 +14923,9 @@ rimraf@~2.4.0:
dependencies:
glob "^6.0.1"
"ringrtc@https://github.com/signalapp/signal-ringrtc-node.git#de96fa13f04e846bdcb0ae9be8e2dc80a932f2be":
version "2.4.2"
resolved "https://github.com/signalapp/signal-ringrtc-node.git#de96fa13f04e846bdcb0ae9be8e2dc80a932f2be"
"ringrtc@https://github.com/signalapp/signal-ringrtc-node.git#0cd60f529d8f17734fe19e7195c9fd3c3f4271db":
version "2.5.0"
resolved "https://github.com/signalapp/signal-ringrtc-node.git#0cd60f529d8f17734fe19e7195c9fd3c3f4271db"
ripemd160@^2.0.0, ripemd160@^2.0.1:
version "2.0.1"