From 28d78a79881b975fa63cad6772bd32a9314d6754 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20N=C3=A9grier?= Date: Tue, 18 May 2021 16:38:56 +0200 Subject: [PATCH] Switching MediaManager to using a Svelte store This allows cleaner and more expressive code, especially regarding whether the webcam should be on or off. --- front/src/Phaser/Game/GameScene.ts | 4 +- front/src/Phaser/Login/EnableCameraScene.ts | 51 ++- front/src/Stores/MediaStore.ts | 360 ++++++++++++++++++++ front/src/Stores/PeerStore.ts | 32 ++ front/src/WebRtc/JitsiFactory.ts | 5 + front/src/WebRtc/MediaManager.ts | 14 + front/style/style.css | 2 + 7 files changed, 458 insertions(+), 10 deletions(-) create mode 100644 front/src/Stores/MediaStore.ts create mode 100644 front/src/Stores/PeerStore.ts diff --git a/front/src/Phaser/Game/GameScene.ts b/front/src/Phaser/Game/GameScene.ts index a5b719e5..7bbf1226 100644 --- a/front/src/Phaser/Game/GameScene.ts +++ b/front/src/Phaser/Game/GameScene.ts @@ -92,6 +92,7 @@ import {PinchManager} from "../UserInput/PinchManager"; import {joystickBaseImg, joystickBaseKey, joystickThumbImg, joystickThumbKey} from "../Components/MobileJoystick"; import {DEPTH_OVERLAY_INDEX} from "./DepthIndexes"; import {waScaleManager} from "../Services/WaScaleManager"; +import {peerStore} from "../../Stores/PeerStore"; import {EmoteManager} from "./EmoteManager"; export interface GameSceneInitInterface { @@ -516,7 +517,7 @@ export class GameScene extends DirtyScene implements CenterListener { } document.addEventListener('visibilitychange', this.onVisibilityChangeCallback); - + this.emoteManager = new EmoteManager(this); } @@ -622,6 +623,7 @@ export class GameScene extends DirtyScene implements CenterListener { // When connection is performed, let's connect SimplePeer this.simplePeer = new SimplePeer(this.connection, !this.room.isPublic, this.playerName); + peerStore.connectToSimplePeer(this.simplePeer); this.GlobalMessageManager = new GlobalMessageManager(this.connection); userMessageManager.setReceiveBanListener(this.bannedUser.bind(this)); diff --git a/front/src/Phaser/Login/EnableCameraScene.ts b/front/src/Phaser/Login/EnableCameraScene.ts index 755ac9a0..6002da7b 100644 --- a/front/src/Phaser/Login/EnableCameraScene.ts +++ b/front/src/Phaser/Login/EnableCameraScene.ts @@ -10,6 +10,14 @@ import {PinchManager} from "../UserInput/PinchManager"; import Zone = Phaser.GameObjects.Zone; import { MenuScene } from "../Menu/MenuScene"; import {ResizableScene} from "./ResizableScene"; +import { + audioConstraintStore, + enableCameraSceneVisibilityStore, + localStreamStore, + mediaStreamConstraintsStore, + videoConstraintStore +} from "../../Stores/MediaStore"; +import type {Unsubscriber} from "svelte/store"; export const EnableCameraSceneName = "EnableCameraScene"; enum LoginTextures { @@ -40,6 +48,7 @@ export class EnableCameraScene extends ResizableScene { private enableCameraSceneElement!: Phaser.GameObjects.DOMElement; private mobileTapZone!: Zone; + private localStreamStoreUnsubscriber!: Unsubscriber; constructor() { super({ @@ -119,9 +128,20 @@ export class EnableCameraScene extends ResizableScene { HtmlUtils.getElementByIdOrFail('webRtcSetup').classList.add('active'); - const mediaPromise = mediaManager.getCamera(); + this.localStreamStoreUnsubscriber = localStreamStore.subscribe((result) => { + if (result.type === 'error') { + // TODO: proper handling of the error + throw result.error; + } + + this.getDevices(); + if (result.stream !== null) { + this.setupStream(result.stream); + } + }); + /*const mediaPromise = mediaManager.getCamera(); mediaPromise.then(this.getDevices.bind(this)); - mediaPromise.then(this.setupStream.bind(this)); + mediaPromise.then(this.setupStream.bind(this));*/ this.input.keyboard.on('keydown-RIGHT', this.nextCam.bind(this)); this.input.keyboard.on('keydown-LEFT', this.previousCam.bind(this)); @@ -133,6 +153,8 @@ export class EnableCameraScene extends ResizableScene { this.add.existing(this.soundMeterSprite); this.onResize(); + + enableCameraSceneVisibilityStore.showEnableCameraScene(); } private previousCam(): void { @@ -140,7 +162,9 @@ export class EnableCameraScene extends ResizableScene { return; } this.cameraSelected--; - mediaManager.setCamera(this.camerasList[this.cameraSelected].deviceId).then(this.setupStream.bind(this)); + videoConstraintStore.setDeviceId(this.camerasList[this.cameraSelected].deviceId); + + //mediaManager.setCamera(this.camerasList[this.cameraSelected].deviceId).then(this.setupStream.bind(this)); } private nextCam(): void { @@ -148,8 +172,10 @@ export class EnableCameraScene extends ResizableScene { return; } this.cameraSelected++; + videoConstraintStore.setDeviceId(this.camerasList[this.cameraSelected].deviceId); + // TODO: the change of camera should be OBSERVED (reactive) - mediaManager.setCamera(this.camerasList[this.cameraSelected].deviceId).then(this.setupStream.bind(this)); + //mediaManager.setCamera(this.camerasList[this.cameraSelected].deviceId).then(this.setupStream.bind(this)); } private previousMic(): void { @@ -157,7 +183,8 @@ export class EnableCameraScene extends ResizableScene { return; } this.microphoneSelected--; - mediaManager.setMicrophone(this.microphonesList[this.microphoneSelected].deviceId).then(this.setupStream.bind(this)); + audioConstraintStore.setDeviceId(this.microphonesList[this.microphoneSelected].deviceId); + //mediaManager.setMicrophone(this.microphonesList[this.microphoneSelected].deviceId).then(this.setupStream.bind(this)); } private nextMic(): void { @@ -165,8 +192,9 @@ export class EnableCameraScene extends ResizableScene { return; } this.microphoneSelected++; + audioConstraintStore.setDeviceId(this.microphonesList[this.microphoneSelected].deviceId); // TODO: the change of camera should be OBSERVED (reactive) - mediaManager.setMicrophone(this.microphonesList[this.microphoneSelected].deviceId).then(this.setupStream.bind(this)); + //mediaManager.setMicrophone(this.microphonesList[this.microphoneSelected].deviceId).then(this.setupStream.bind(this)); } /** @@ -260,15 +288,20 @@ export class EnableCameraScene extends ResizableScene { HtmlUtils.getElementByIdOrFail('webRtcSetup').style.display = 'none'; this.soundMeter.stop(); - mediaManager.stopCamera(); - mediaManager.stopMicrophone(); + enableCameraSceneVisibilityStore.hideEnableCameraScene(); + this.localStreamStoreUnsubscriber(); + //mediaManager.stopCamera(); + //mediaManager.stopMicrophone(); - this.scene.sleep(EnableCameraSceneName) + this.scene.sleep(EnableCameraSceneName); gameManager.goToStartingMap(this.scene); } private async getDevices() { + // TODO: switch this in a store. const mediaDeviceInfos = await navigator.mediaDevices.enumerateDevices(); + this.microphonesList = []; + this.camerasList = []; for (const mediaDeviceInfo of mediaDeviceInfos) { if (mediaDeviceInfo.kind === 'audioinput') { this.microphonesList.push(mediaDeviceInfo); diff --git a/front/src/Stores/MediaStore.ts b/front/src/Stores/MediaStore.ts new file mode 100644 index 00000000..e0f351f2 --- /dev/null +++ b/front/src/Stores/MediaStore.ts @@ -0,0 +1,360 @@ +import {derived, Readable, readable, writable, Writable} from "svelte/store"; +import {peerStore} from "./PeerStore"; +import {localUserStore} from "../Connexion/LocalUserStore"; +import {ITiledMapGroupLayer, ITiledMapObjectLayer, ITiledMapTileLayer} from "../Phaser/Map/ITiledMap"; + +/** + * A store that contains the camera state requested by the user (on or off). + */ +function createRequestedCameraState() { + const { subscribe, set, update } = writable(true); + + return { + subscribe, + enableWebcam: () => set(true), + disableWebcam: () => set(false), + }; +} + +/** + * A store that contains the microphone state requested by the user (on or off). + */ +function createRequestedMicrophoneState() { + const { subscribe, set, update } = writable(true); + + return { + subscribe, + enableMicrophone: () => set(true), + disableMicrophone: () => set(false), + }; +} + +/** + * A store containing whether the current page is visible or not. + */ +export const visibilityStore = readable(document.visibilityState === 'visible', function start(set) { + const onVisibilityChange = () => { + set(document.visibilityState === 'visible'); + }; + + document.addEventListener('visibilitychange', onVisibilityChange); + + return function stop() { + document.removeEventListener('visibilitychange', onVisibilityChange); + }; +}); + +/** + * A store that contains whether the game overlay is shown or not. + * Typically, the overlay is hidden when entering Jitsi meet. + */ +function createGameOverlayVisibilityStore() { + const { subscribe, set, update } = writable(false); + + return { + subscribe, + showGameOverlay: () => set(true), + hideGameOverlay: () => set(false), + }; +} + +/** + * A store that contains whether the EnableCameraScene is shown or not. + */ +function createEnableCameraSceneVisibilityStore() { + const { subscribe, set, update } = writable(false); + + return { + subscribe, + showEnableCameraScene: () => set(true), + hideEnableCameraScene: () => set(false), + }; +} + +export const requestedCameraState = createRequestedCameraState(); +export const requestedMicrophoneState = createRequestedMicrophoneState(); +export const gameOverlayVisibilityStore = createGameOverlayVisibilityStore(); +export const enableCameraSceneVisibilityStore = createEnableCameraSceneVisibilityStore(); + +/** + * A store that contains video constraints. + */ +function createVideoConstraintStore() { + const { subscribe, set, update } = writable({ + width: { min: 640, ideal: 1280, max: 1920 }, + height: { min: 400, ideal: 720 }, + frameRate: { ideal: localUserStore.getVideoQualityValue() }, + facingMode: "user", + resizeMode: 'crop-and-scale', + aspectRatio: 1.777777778 + } as boolean|MediaTrackConstraints); + + let selectedDeviceId = null; + + return { + subscribe, + setDeviceId: (deviceId: string) => update((constraints) => { + selectedDeviceId = deviceId; + + if (typeof(constraints) === 'boolean') { + constraints = {} + } + constraints.deviceId = { + exact: selectedDeviceId + }; + + return constraints; + }) + }; +} + +export const videoConstraintStore = createVideoConstraintStore(); + +/** + * A store that contains video constraints. + */ +function createAudioConstraintStore() { + const { subscribe, set, update } = writable({ + //TODO: make these values configurable in the game settings menu and store them in localstorage + autoGainControl: false, + echoCancellation: true, + noiseSuppression: true + } as boolean|MediaTrackConstraints); + + let selectedDeviceId = null; + + return { + subscribe, + setDeviceId: (deviceId: string) => update((constraints) => { + selectedDeviceId = deviceId; + + if (typeof(constraints) === 'boolean') { + constraints = {} + } + constraints.deviceId = { + exact: selectedDeviceId + }; + + return constraints; + }) + }; +} + +export const audioConstraintStore = createAudioConstraintStore(); + + +let timeout: NodeJS.Timeout; + +/** + * A store containing the media constraints we want to apply. + */ +export const mediaStreamConstraintsStore = derived( + [ + requestedCameraState, + requestedMicrophoneState, + visibilityStore, + gameOverlayVisibilityStore, + peerStore, + enableCameraSceneVisibilityStore, + videoConstraintStore, + audioConstraintStore, + ], ( + [ + $requestedCameraState, + $requestedMicrophoneState, + $visibilityStore, + $gameOverlayVisibilityStore, + $peerStore, + $enableCameraSceneVisibilityStore, + $videoConstraintStore, + $audioConstraintStore, + ], set + ) => { + let currentVideoConstraint: boolean|MediaTrackConstraints = $videoConstraintStore; + let currentAudioConstraint: boolean|MediaTrackConstraints = $audioConstraintStore; + + if ($enableCameraSceneVisibilityStore) { + set({ + video: currentVideoConstraint, + audio: currentAudioConstraint, + }); + return; + } + + // Disable webcam if the user requested so + if ($requestedCameraState === false) { + currentVideoConstraint = false; + } + + // Disable microphone if the user requested so + if ($requestedMicrophoneState === false) { + currentAudioConstraint = false; + } + + // Disable webcam and microphone when in a Jitsi + if ($gameOverlayVisibilityStore === false) { + currentVideoConstraint = false; + currentAudioConstraint = false; + } + + // Disable webcam if the game is not visible and we are talking to noone. + if ($visibilityStore === false && $peerStore.size === 0) { + currentVideoConstraint = false; + } + + if (timeout) { + clearTimeout(timeout); + } + + // Let's wait a little bit to avoid sending too many constraint changes. + timeout = setTimeout(() => { + set({ + video: currentVideoConstraint, + audio: currentAudioConstraint, + }); + }, 100) +}, { + video: false, + audio: false +} as MediaStreamConstraints); + +export type LocalStreamStoreValue = StreamSuccessValue | StreamErrorValue; + +interface StreamSuccessValue { + type: "success", + stream: MediaStream|null, + // The constraints that we got (and not the one that have been requested) + constraints: MediaStreamConstraints +} + +interface StreamErrorValue { + type: "error", + error: Error, + constraints: MediaStreamConstraints +} + +let currentStream : MediaStream|null = null; + +/** + * Stops the camera from filming + */ +function stopCamera(): void { + if (currentStream) { + for (const track of currentStream.getVideoTracks()) { + track.stop(); + } + } +} + +/** + * Stops the microphone from listening + */ +function stopMicrophone(): void { + if (currentStream) { + for (const track of currentStream.getAudioTracks()) { + track.stop(); + } + } +} + +/** + * A store containing the MediaStream object (or null if nothing requested, or Error if an error occurred) + */ +export const localStreamStore = derived, LocalStreamStoreValue>(mediaStreamConstraintsStore, ($mediaStreamConstraintsStore, set) => { + const constraints = { ...$mediaStreamConstraintsStore }; + + if (navigator.mediaDevices === undefined) { + if (window.location.protocol === 'http:') { + //throw new Error('Unable to access your camera or microphone. You need to use a HTTPS connection.'); + set({ + type: 'error', + error: new Error('Unable to access your camera or microphone. You need to use a HTTPS connection.'), + constraints + }); + } else { + //throw new Error('Unable to access your camera or microphone. Your browser is too old.'); + set({ + type: 'error', + error: new Error('Unable to access your camera or microphone. Your browser is too old.'), + constraints + }); + } + } + + if (constraints.audio === false) { + stopMicrophone(); + } + if (constraints.video === false) { + stopCamera(); + } + + if (constraints.audio === false && constraints.video === false) { + set({ + type: 'success', + stream: null, + constraints + }); + return; + } + + (async () => { + try { + currentStream = await navigator.mediaDevices.getUserMedia(constraints); + set({ + type: 'success', + stream: currentStream, + constraints + }); + return; + } catch (e) { + if (constraints.video !== false) { + console.info("Error. Unable to get microphone and/or camera access. Trying audio only.", $mediaStreamConstraintsStore, e); + // TODO: does it make sense to pop this error when retrying? + set({ + type: 'error', + error: e, + constraints + }); + // Let's try without video constraints + requestedCameraState.disableWebcam(); + } else { + console.info("Error. Unable to get microphone and/or camera access.", $mediaStreamConstraintsStore, e); + set({ + type: 'error', + error: e, + constraints + }); + } + + /*constraints.video = false; + if (constraints.audio === false) { + console.info("Error. Unable to get microphone and/or camera access.", $mediaStreamConstraintsStore, e); + set({ + type: 'error', + error: e, + constraints + }); + // Let's make as if the user did not ask. + requestedCameraState.disableWebcam(); + } else { + console.info("Error. Unable to get microphone and/or camera access. Trying audio only.", $mediaStreamConstraintsStore, e); + try { + currentStream = await navigator.mediaDevices.getUserMedia(constraints); + set({ + type: 'success', + stream: currentStream, + constraints + }); + return; + } catch (e2) { + console.info("Error. Unable to get microphone fallback access.", $mediaStreamConstraintsStore, e2); + set({ + type: 'error', + error: e, + constraints + }); + } + }*/ + } + })(); +}); diff --git a/front/src/Stores/PeerStore.ts b/front/src/Stores/PeerStore.ts new file mode 100644 index 00000000..14d14754 --- /dev/null +++ b/front/src/Stores/PeerStore.ts @@ -0,0 +1,32 @@ +import { derived, writable, Writable } from "svelte/store"; +import type {UserSimplePeerInterface} from "../WebRtc/SimplePeer"; +import type {SimplePeer} from "../WebRtc/SimplePeer"; + +/** + * A store that contains the camera state requested by the user (on or off). + */ +function createPeerStore() { + let users = new Map(); + + const { subscribe, set, update } = writable(users); + + return { + subscribe, + connectToSimplePeer: (simplePeer: SimplePeer) => { + users = new Map(); + set(users); + simplePeer.registerPeerConnectionListener({ + onConnect(user: UserSimplePeerInterface) { + users.set(user.userId, user); + set(users); + }, + onDisconnect(userId: number) { + users.delete(userId); + set(users); + } + }) + } + }; +} + +export const peerStore = createPeerStore(); diff --git a/front/src/WebRtc/JitsiFactory.ts b/front/src/WebRtc/JitsiFactory.ts index 8ddbba7b..4e70a4d2 100644 --- a/front/src/WebRtc/JitsiFactory.ts +++ b/front/src/WebRtc/JitsiFactory.ts @@ -1,6 +1,7 @@ import {JITSI_URL} from "../Enum/EnvironmentVariable"; import {mediaManager} from "./MediaManager"; import {coWebsiteManager} from "./CoWebsiteManager"; +import {requestedCameraState, requestedMicrophoneState} from "../Stores/MediaStore"; declare const window:any; // eslint-disable-line @typescript-eslint/no-explicit-any interface jitsiConfigInterface { @@ -138,14 +139,18 @@ class JitsiFactory { //restore previous config if(this.previousConfigMeet?.startWithAudioMuted){ await mediaManager.disableMicrophone(); + requestedMicrophoneState.disableMicrophone(); }else{ await mediaManager.enableMicrophone(); + requestedMicrophoneState.enableMicrophone(); } if(this.previousConfigMeet?.startWithVideoMuted){ await mediaManager.disableCamera(); + requestedCameraState.disableWebcam(); }else{ await mediaManager.enableCamera(); + requestedCameraState.enableWebcam(); } } diff --git a/front/src/WebRtc/MediaManager.ts b/front/src/WebRtc/MediaManager.ts index b7594670..e604c50f 100644 --- a/front/src/WebRtc/MediaManager.ts +++ b/front/src/WebRtc/MediaManager.ts @@ -6,6 +6,12 @@ import {localUserStore} from "../Connexion/LocalUserStore"; import type {UserSimplePeerInterface} from "./SimplePeer"; import {SoundMeter} from "../Phaser/Components/SoundMeter"; import {DISABLE_NOTIFICATIONS} from "../Enum/EnvironmentVariable"; +import { + gameOverlayVisibilityStore, + mediaStreamConstraintsStore, + requestedCameraState, + requestedMicrophoneState +} from "../Stores/MediaStore"; declare const navigator:any; // eslint-disable-line @typescript-eslint/no-explicit-any @@ -90,12 +96,14 @@ export class MediaManager { e.preventDefault(); this.enableMicrophone(); //update tracking + requestedMicrophoneState.enableMicrophone(); }); this.microphone = HtmlUtils.getElementByIdOrFail('microphone'); this.microphone.addEventListener('click', (e: MouseEvent) => { e.preventDefault(); this.disableMicrophone(); //update tracking + requestedMicrophoneState.disableMicrophone(); }); this.cinemaBtn = HtmlUtils.getElementByIdOrFail('btn-video'); @@ -105,12 +113,14 @@ export class MediaManager { e.preventDefault(); this.enableCamera(); //update tracking + requestedCameraState.enableWebcam(); }); this.cinema = HtmlUtils.getElementByIdOrFail('cinema'); this.cinema.addEventListener('click', (e: MouseEvent) => { e.preventDefault(); this.disableCamera(); //update tracking + requestedCameraState.disableWebcam(); }); this.monitorBtn = HtmlUtils.getElementByIdOrFail('btn-monitor'); @@ -214,6 +224,8 @@ export class MediaManager { this.triggerCloseJitsiFrameButton(); } buttonCloseFrame.removeEventListener('click', functionTrigger); + + gameOverlayVisibilityStore.showGameOverlay(); } public hideGameOverlay(): void { @@ -225,6 +237,8 @@ export class MediaManager { this.triggerCloseJitsiFrameButton(); } buttonCloseFrame.addEventListener('click', functionTrigger); + + gameOverlayVisibilityStore.hideGameOverlay(); } public isGameOverlayVisible(): boolean { diff --git a/front/style/style.css b/front/style/style.css index d95ac701..d6b5c433 100644 --- a/front/style/style.css +++ b/front/style/style.css @@ -346,6 +346,8 @@ video#myCamVideo{ #myCamVideoSetup { width: 100%; height: 100%; + -webkit-transform: scaleX(-1); + transform: scaleX(-1); } .webrtcsetup.active{ display: block;