diff --git a/back/src/Controller/IoSocketController.ts b/back/src/Controller/IoSocketController.ts index 28dd2da2..1d5bbe06 100644 --- a/back/src/Controller/IoSocketController.ts +++ b/back/src/Controller/IoSocketController.ts @@ -13,7 +13,6 @@ import {MessageUserJoined} from "../Model/Websocket/MessageUserJoined"; import {MessageUserMoved} from "../Model/Websocket/MessageUserMoved"; import si from "systeminformation"; import {Gauge} from "prom-client"; -import os from 'os'; import {TokenInterface} from "../Controller/AuthenticateController"; import {isJoinRoomMessageInterface} from "../Model/Websocket/JoinRoomMessage"; import {isPointInterface, PointInterface} from "../Model/Websocket/PointInterface"; @@ -28,6 +27,7 @@ enum SockerIoEvent { USER_MOVED = "user-moved", // From server to client USER_LEFT = "user-left", // From server to client WEBRTC_SIGNAL = "webrtc-signal", + WEBRTC_SCREEN_SHARING_SIGNAL = "webrtc-screen-sharing-signal", WEBRTC_START = "webrtc-start", WEBRTC_DISCONNECT = "webrtc-disconect", MESSAGE_ERROR = "message-error", @@ -226,18 +226,11 @@ export class IoSocketController { }); socket.on(SockerIoEvent.WEBRTC_SIGNAL, (data: unknown) => { - if (!isWebRtcSignalMessageInterface(data)) { - socket.emit(SockerIoEvent.MESSAGE_ERROR, {message: 'Invalid WEBRTC_SIGNAL message.'}); - console.warn('Invalid WEBRTC_SIGNAL message received: ', data); - return; - } - //send only at user - const client = this.sockets.get(data.receiverId); - if (client === undefined) { - console.warn("While exchanging a WebRTC signal: client with id ", data.receiverId, " does not exist. This might be a race condition."); - return; - } - return client.emit(SockerIoEvent.WEBRTC_SIGNAL, data); + this.emitVideo((socket as ExSocketInterface), data); + }); + + socket.on(SockerIoEvent.WEBRTC_SCREEN_SHARING_SIGNAL, (data: unknown) => { + this.emitScreenSharing((socket as ExSocketInterface), data); }); socket.on(SockerIoEvent.DISCONNECT, () => { @@ -284,6 +277,42 @@ export class IoSocketController { }); } + emitVideo(socket: ExSocketInterface, data: unknown){ + if (!isWebRtcSignalMessageInterface(data)) { + socket.emit(SockerIoEvent.MESSAGE_ERROR, {message: 'Invalid WEBRTC_SIGNAL message.'}); + console.warn('Invalid WEBRTC_SIGNAL message received: ', data); + return; + } + //send only at user + const client = this.sockets.get(data.receiverId); + if (client === undefined) { + console.warn("While exchanging a WebRTC signal: client with id ", data.receiverId, " does not exist. This might be a race condition."); + return; + } + return client.emit(SockerIoEvent.WEBRTC_SIGNAL, { + userId: socket.userId, + signal: data.signal + }); + } + + emitScreenSharing(socket: ExSocketInterface, data: unknown){ + if (!isWebRtcSignalMessageInterface(data)) { + socket.emit(SockerIoEvent.MESSAGE_ERROR, {message: 'Invalid WEBRTC_SCREEN_SHARING message.'}); + console.warn('Invalid WEBRTC_SCREEN_SHARING message received: ', data); + return; + } + //send only at user + const client = this.sockets.get(data.receiverId); + if (client === undefined) { + console.warn("While exchanging a WEBRTC_SCREEN_SHARING signal: client with id ", data.receiverId, " does not exist. This might be a race condition."); + return; + } + return client.emit(SockerIoEvent.WEBRTC_SCREEN_SHARING_SIGNAL, { + userId: socket.userId, + signal: data.signal + }); + } + searchClientByIdOrFail(userId: string): ExSocketInterface { const client: ExSocketInterface|undefined = this.sockets.get(userId); if (client === undefined) { @@ -364,13 +393,15 @@ export class IoSocketController { if (this.Io.sockets.adapter.rooms[roomId].length < 2 /*|| this.Io.sockets.adapter.rooms[roomId].length >= 4*/) { return; } + + // TODO: scanning all sockets is maybe not the most efficient const clients: Array = (Object.values(this.Io.sockets.sockets) as Array) .filter((client: ExSocketInterface) => client.webRtcRoomId && client.webRtcRoomId === roomId); //send start at one client to initialise offer webrtc //send all users in room to create PeerConnection in front clients.forEach((client: ExSocketInterface, index: number) => { - const clientsId = clients.reduce((tabs: Array, clientId: ExSocketInterface, indexClientId: number) => { + const peerClients = clients.reduce((tabs: Array, clientId: ExSocketInterface, indexClientId: number) => { if (!clientId.userId || clientId.userId === client.userId) { return tabs; } @@ -382,7 +413,7 @@ export class IoSocketController { return tabs; }, []); - client.emit(SockerIoEvent.WEBRTC_START, {clients: clientsId, roomId: roomId}); + client.emit(SockerIoEvent.WEBRTC_START, {clients: peerClients, roomId: roomId}); }); } diff --git a/back/src/Model/Websocket/WebRtcSignalMessage.ts b/back/src/Model/Websocket/WebRtcSignalMessage.ts index 7edffdfa..5a0dd1af 100644 --- a/back/src/Model/Websocket/WebRtcSignalMessage.ts +++ b/back/src/Model/Websocket/WebRtcSignalMessage.ts @@ -1,10 +1,18 @@ import * as tg from "generic-type-guard"; +export const isSignalData = + new tg.IsInterface().withProperties({ + type: tg.isOptional(tg.isString) + }).get(); + export const isWebRtcSignalMessageInterface = new tg.IsInterface().withProperties({ - userId: tg.isString, receiverId: tg.isString, - roomId: tg.isString, - signal: tg.isUnknown + signal: isSignalData + }).get(); +export const isWebRtcScreenSharingStartMessageInterface = + new tg.IsInterface().withProperties({ + userId: tg.isString, + roomId: tg.isString }).get(); export type WebRtcSignalMessageInterface = tg.GuardedType; diff --git a/front/dist/index.html b/front/dist/index.html index 92a7bf3c..68940afe 100644 --- a/front/dist/index.html +++ b/front/dist/index.html @@ -68,15 +68,19 @@
-
-
- - -
-
- - -
+
+
+
+ + +
+
+ + +
+
+ +
@@ -100,9 +104,15 @@ +
+ + +
--> +
+
diff --git a/front/dist/resources/logos/monitor-close.svg b/front/dist/resources/logos/monitor-close.svg new file mode 100644 index 00000000..80056e2d --- /dev/null +++ b/front/dist/resources/logos/monitor-close.svg @@ -0,0 +1,44 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/front/dist/resources/logos/monitor.svg b/front/dist/resources/logos/monitor.svg new file mode 100644 index 00000000..d4b586c6 --- /dev/null +++ b/front/dist/resources/logos/monitor.svg @@ -0,0 +1,15 @@ + + + + + + diff --git a/front/dist/resources/style/style.css b/front/dist/resources/style/style.css index 30e099ef..382a4444 100644 --- a/front/dist/resources/style/style.css +++ b/front/dist/resources/style/style.css @@ -79,6 +79,13 @@ video#myCamVideo{ } +.btn-cam-action { + position: absolute; + bottom: 0px; + right: 0px; + width: 450px; + height: 150px; +} /*btn animation*/ .btn-cam-action div{ cursor: pointer; @@ -93,7 +100,7 @@ video#myCamVideo{ transition-timing-function: ease-in-out; bottom: 20px; } -#activeCam:hover .btn-cam-action div{ +.btn-cam-action:hover div{ transform: translateY(0); } .btn-cam-action div:hover{ @@ -106,9 +113,13 @@ video#myCamVideo{ right: 44px; } .btn-video{ - transition: all .2s; + transition: all .25s; right: 134px; } +.btn-monitor{ + transition: all .2s; + right: 224px; +} /*.btn-call{ transition: all .1s; left: 0px; diff --git a/front/src/Connection.ts b/front/src/Connection.ts index 04715df6..783b5d41 100644 --- a/front/src/Connection.ts +++ b/front/src/Connection.ts @@ -6,12 +6,12 @@ import {SetPlayerDetailsMessage} from "./Messages/SetPlayerDetailsMessage"; const SocketIo = require('socket.io-client'); import Socket = SocketIOClient.Socket; import {PlayerAnimationNames} from "./Phaser/Player/Animation"; -import {UserSimplePeer} from "./WebRtc/SimplePeer"; +import {UserSimplePeerInterface} from "./WebRtc/SimplePeer"; import {SignalData} from "simple-peer"; - enum EventMessage{ WEBRTC_SIGNAL = "webrtc-signal", + WEBRTC_SCREEN_SHARING_SIGNAL = "webrtc-screen-sharing-signal", WEBRTC_START = "webrtc-start", JOIN_ROOM = "join-room", // bi-directional USER_POSITION = "user-position", // bi-directional @@ -72,17 +72,20 @@ export interface GroupCreatedUpdatedMessageInterface { export interface WebRtcStartMessageInterface { roomId: string, - clients: UserSimplePeer[] + clients: UserSimplePeerInterface[] } export interface WebRtcDisconnectMessageInterface { userId: string } -export interface WebRtcSignalMessageInterface { - userId: string, +export interface WebRtcSignalSentMessageInterface { receiverId: string, - roomId: string, + signal: SignalData +} + +export interface WebRtcSignalReceivedMessageInterface { + userId: string, signal: SignalData } @@ -188,23 +191,32 @@ export class Connection implements Connection { this.socket.on(EventMessage.CONNECT_ERROR, callback) } - public sendWebrtcSignal(signal: unknown, roomId: string, userId? : string|null, receiverId? : string) { + public sendWebrtcSignal(signal: unknown, receiverId : string) { return this.socket.emit(EventMessage.WEBRTC_SIGNAL, { - userId: userId ? userId : this.userId, - receiverId: receiverId ? receiverId : this.userId, - roomId: roomId, + receiverId: receiverId, signal: signal - }); + } as WebRtcSignalSentMessageInterface); + } + + public sendWebrtcScreenSharingSignal(signal: unknown, receiverId : string) { + return this.socket.emit(EventMessage.WEBRTC_SCREEN_SHARING_SIGNAL, { + receiverId: receiverId, + signal: signal + } as WebRtcSignalSentMessageInterface); } public receiveWebrtcStart(callback: (message: WebRtcStartMessageInterface) => void) { this.socket.on(EventMessage.WEBRTC_START, callback); } - public receiveWebrtcSignal(callback: (message: WebRtcSignalMessageInterface) => void) { + public receiveWebrtcSignal(callback: (message: WebRtcSignalReceivedMessageInterface) => void) { return this.socket.on(EventMessage.WEBRTC_SIGNAL, callback); } + public receiveWebrtcScreenSharingSignal(callback: (message: WebRtcSignalReceivedMessageInterface) => void) { + return this.socket.on(EventMessage.WEBRTC_SCREEN_SHARING_SIGNAL, callback); + } + public onServerDisconnected(callback: (reason: string) => void): void { this.socket.on('disconnect', (reason: string) => { if (reason === 'io client disconnect') { diff --git a/front/src/Phaser/Game/GameScene.ts b/front/src/Phaser/Game/GameScene.ts index c4517545..3c3a6536 100644 --- a/front/src/Phaser/Game/GameScene.ts +++ b/front/src/Phaser/Game/GameScene.ts @@ -18,7 +18,7 @@ import {PlayerMovement} from "./PlayerMovement"; import {PlayersPositionInterpolator} from "./PlayersPositionInterpolator"; import {RemotePlayer} from "../Entity/RemotePlayer"; import {Queue} from 'queue-typescript'; -import {SimplePeer, UserSimplePeer} from "../../WebRtc/SimplePeer"; +import {SimplePeer, UserSimplePeerInterface} from "../../WebRtc/SimplePeer"; import {ReconnectingSceneName} from "../Reconnecting/ReconnectingScene"; import {FourOFourSceneName} from "../Reconnecting/FourOFourScene"; import {loadAllLayers} from "../Entity/body_character"; @@ -229,7 +229,7 @@ export class GameScene extends Phaser.Scene { this.simplePeer = new SimplePeer(this.connection); const self = this; this.simplePeer.registerPeerConnectionListener({ - onConnect(user: UserSimplePeer) { + onConnect(user: UserSimplePeerInterface) { self.presentationModeSprite.setVisible(true); self.chatModeSprite.setVisible(true); }, diff --git a/front/src/WebRtc/MediaManager.ts b/front/src/WebRtc/MediaManager.ts index 39a61738..314fea14 100644 --- a/front/src/WebRtc/MediaManager.ts +++ b/front/src/WebRtc/MediaManager.ts @@ -1,4 +1,5 @@ import {DivImportance, layoutManager} from "./LayoutManager"; +import {HtmlUtils} from "./HtmlUtils"; const videoConstraint: boolean|MediaTrackConstraints = { width: { ideal: 1280 }, @@ -7,15 +8,20 @@ const videoConstraint: boolean|MediaTrackConstraints = { }; type UpdatedLocalStreamCallback = (media: MediaStream) => void; +type StartScreenSharingCallback = (media: MediaStream) => void; +type StopScreenSharingCallback = (media: MediaStream) => void; // TODO: Split MediaManager in 2 classes: MediaManagerUI (in charge of HTML) and MediaManager (singleton in charge of the camera only) // TODO: verify that microphone event listeners are not triggered plenty of time NOW (since MediaManager is created many times!!!!) export class MediaManager { localStream: MediaStream|null = null; + localScreenCapture: MediaStream|null = null; private remoteVideo: Map = new Map(); myCamVideo: HTMLVideoElement; cinemaClose: HTMLImageElement; cinema: HTMLImageElement; + monitorClose: HTMLImageElement; + monitor: HTMLImageElement; microphoneClose: HTMLImageElement; microphone: HTMLImageElement; webrtcInAudio: HTMLAudioElement; @@ -24,8 +30,12 @@ export class MediaManager { video: videoConstraint }; updatedLocalStreamCallBacks : Set = new Set(); + startScreenSharingCallBacks : Set = new Set(); + stopScreenSharingCallBacks : Set = new Set(); + constructor() { + this.myCamVideo = this.getElementByIdOrFail('myCamVideo'); this.webrtcInAudio = this.getElementByIdOrFail('audio-webrtc-in'); this.webrtcInAudio.volume = 0.2; @@ -34,13 +44,13 @@ export class MediaManager { this.microphoneClose.style.display = "none"; this.microphoneClose.addEventListener('click', (e: MouseEvent) => { e.preventDefault(); - this.enabledMicrophone(); + this.enableMicrophone(); //update tracking }); this.microphone = this.getElementByIdOrFail('microphone'); this.microphone.addEventListener('click', (e: MouseEvent) => { e.preventDefault(); - this.disabledMicrophone(); + this.disableMicrophone(); //update tracking }); @@ -48,22 +58,47 @@ export class MediaManager { this.cinemaClose.style.display = "none"; this.cinemaClose.addEventListener('click', (e: MouseEvent) => { e.preventDefault(); - this.enabledCamera(); + this.enableCamera(); //update tracking }); this.cinema = this.getElementByIdOrFail('cinema'); this.cinema.addEventListener('click', (e: MouseEvent) => { e.preventDefault(); - this.disabledCamera(); + this.disableCamera(); + //update tracking + }); + + this.monitorClose = this.getElementByIdOrFail('monitor-close'); + this.monitorClose.style.display = "block"; + this.monitorClose.addEventListener('click', (e: MouseEvent) => { + e.preventDefault(); + this.enableScreenSharing(); + //update tracking + }); + this.monitor = this.getElementByIdOrFail('monitor'); + this.monitor.style.display = "none"; + this.monitor.addEventListener('click', (e: MouseEvent) => { + e.preventDefault(); + this.disableScreenSharing(); //update tracking }); } - onUpdateLocalStream(callback: UpdatedLocalStreamCallback): void { + public onUpdateLocalStream(callback: UpdatedLocalStreamCallback): void { this.updatedLocalStreamCallBacks.add(callback); } + public onStartScreenSharing(callback: StartScreenSharingCallback): void { + + this.startScreenSharingCallBacks.add(callback); + } + + public onStopScreenSharing(callback: StopScreenSharingCallback): void { + + this.stopScreenSharingCallBacks.add(callback); + } + removeUpdateLocalStreamEventListener(callback: UpdatedLocalStreamCallback): void { this.updatedLocalStreamCallBacks.delete(callback); } @@ -74,12 +109,24 @@ export class MediaManager { } } - activeVisio(){ + private triggerStartedScreenSharingCallbacks(stream: MediaStream): void { + for (const callback of this.startScreenSharingCallBacks) { + callback(stream); + } + } + + private triggerStoppedScreenSharingCallbacks(stream: MediaStream): void { + for (const callback of this.stopScreenSharingCallBacks) { + callback(stream); + } + } + + showGameOverlay(){ const gameOverlay = this.getElementByIdOrFail('game-overlay'); gameOverlay.classList.add('active'); } - enabledCamera() { + private enableCamera() { this.cinemaClose.style.display = "none"; this.cinema.style.display = "block"; this.constraintsMedia.video = videoConstraint; @@ -88,7 +135,7 @@ export class MediaManager { }); } - disabledCamera() { + private disableCamera() { this.cinemaClose.style.display = "block"; this.cinema.style.display = "none"; this.constraintsMedia.video = false; @@ -103,7 +150,7 @@ export class MediaManager { }); } - enabledMicrophone() { + private enableMicrophone() { this.microphoneClose.style.display = "none"; this.microphone.style.display = "block"; this.constraintsMedia.audio = true; @@ -112,7 +159,7 @@ export class MediaManager { }); } - disabledMicrophone() { + private disableMicrophone() { this.microphoneClose.style.display = "block"; this.microphone.style.display = "none"; this.constraintsMedia.audio = false; @@ -126,6 +173,78 @@ export class MediaManager { }); } + private enableScreenSharing() { + this.monitorClose.style.display = "none"; + this.monitor.style.display = "block"; + this.getScreenMedia().then((stream) => { + this.triggerStartedScreenSharingCallbacks(stream); + }); + } + + private disableScreenSharing() { + this.monitorClose.style.display = "block"; + this.monitor.style.display = "none"; + this.removeActiveScreenSharingVideo('me'); + this.localScreenCapture?.getTracks().forEach((track: MediaStreamTrack) => { + track.stop(); + }); + if (this.localScreenCapture === null) { + console.warn('Weird: trying to remove a screen sharing that is not enabled'); + return; + } + const localScreenCapture = this.localScreenCapture; + this.getCamera().then((stream) => { + this.triggerStoppedScreenSharingCallbacks(localScreenCapture); + }); + this.localScreenCapture = null; + } + + //get screen + getScreenMedia() : Promise{ + try { + return this._startScreenCapture() + .then((stream: MediaStream) => { + this.localScreenCapture = stream; + + // If stream ends (for instance if user clicks the stop screen sharing button in the browser), let's close the view + for (const track of stream.getTracks()) { + track.onended = () => { + this.disableScreenSharing(); + }; + } + + this.addScreenSharingActiveVideo('me', DivImportance.Normal); + HtmlUtils.getElementByIdOrFail('screen-sharing-me').srcObject = stream; + + return stream; + }) + .catch((err: unknown) => { + console.error("Error => getScreenMedia => ", err); + throw err; + }); + }catch (err) { + return new Promise((resolve, reject) => { // eslint-disable-line no-unused-vars + reject(err); + }); + } + } + + private _startScreenCapture() { + // getDisplayMedia was moved to mediaDevices in 2018. Typescript definitions are not up to date yet. + // See: https://github.com/w3c/mediacapture-screen-share/pull/86 + // https://github.com/microsoft/TypeScript/issues/31821 + if ((navigator as any).getDisplayMedia) { // eslint-disable-line @typescript-eslint/no-explicit-any + return (navigator as any).getDisplayMedia({video: true}); // eslint-disable-line @typescript-eslint/no-explicit-any + } else if ((navigator.mediaDevices as any).getDisplayMedia) { // eslint-disable-line @typescript-eslint/no-explicit-any + return (navigator.mediaDevices as any).getDisplayMedia({video: true}); // eslint-disable-line @typescript-eslint/no-explicit-any + } else { + //return navigator.mediaDevices.getUserMedia(({video: {mediaSource: 'screen'}} as any)); + return new Promise((resolve, reject) => { // eslint-disable-line no-unused-vars + reject("error sharing screen"); + }); + } + } + //get camera async getCamera(): Promise { if (navigator.mediaDevices === undefined) { @@ -205,6 +324,25 @@ export class MediaManager { this.remoteVideo.set(userId, this.getElementByIdOrFail(userId)); } + /** + * + * @param userId + */ + addScreenSharingActiveVideo(userId : string, divImportance: DivImportance = DivImportance.Important){ + //this.webrtcInAudio.play(); + + userId = `screen-sharing-${userId}`; + const html = ` +
+ +
+ `; + + layoutManager.add(divImportance, userId, html); + + this.remoteVideo.set(userId, this.getElementByIdOrFail(userId)); + } + /** * * @param userId @@ -272,6 +410,15 @@ export class MediaManager { } remoteVideo.srcObject = stream; } + addStreamRemoteScreenSharing(userId : string, stream : MediaStream){ + // In the case of screen sharing (going both ways), we may need to create the HTML element if it does not exist yet + const remoteVideo = this.remoteVideo.get(`screen-sharing-${userId}`); + if (remoteVideo === undefined) { + this.addScreenSharingActiveVideo(userId); + } + + this.addStreamRemoteVideo(`screen-sharing-${userId}`, stream); + } /** * @@ -281,6 +428,9 @@ export class MediaManager { layoutManager.remove(userId); this.remoteVideo.delete(userId); } + removeActiveScreenSharingVideo(userId : string) { + this.removeActiveVideo(`screen-sharing-${userId}`) + } isConnecting(userId : string): void { const connectingSpinnerDiv = this.getSpinner(userId); @@ -299,6 +449,7 @@ export class MediaManager { } isError(userId : string): void { + console.log("isError", `div-${userId}`); const element = document.getElementById(`div-${userId}`); if(!element){ return; @@ -309,6 +460,10 @@ export class MediaManager { } errorDiv.style.display = 'block'; } + isErrorScreenSharing(userId : string): void { + this.isError(`screen-sharing-${userId}`); + } + private getSpinner(userId : string): HTMLDivElement|null { const element = document.getElementById(`div-${userId}`); diff --git a/front/src/WebRtc/ScreenSharingPeer.ts b/front/src/WebRtc/ScreenSharingPeer.ts new file mode 100644 index 00000000..35f43201 --- /dev/null +++ b/front/src/WebRtc/ScreenSharingPeer.ts @@ -0,0 +1,127 @@ +import * as SimplePeerNamespace from "simple-peer"; +import {mediaManager} from "./MediaManager"; +import {Connection} from "../Connection"; + +const Peer: SimplePeerNamespace.SimplePeer = require('simple-peer'); + +/** + * A peer connection used to transmit video / audio signals between 2 peers. + */ +export class ScreenSharingPeer extends Peer { + /** + * Whether this connection is currently receiving a video stream from a remote user. + */ + private isReceivingStream:boolean = false; + + constructor(private userId: string, initiator: boolean, private connection: Connection) { + super({ + initiator: initiator ? initiator : false, + reconnectTimer: 10000, + config: { + iceServers: [ + { + urls: 'stun:stun.l.google.com:19302' + }, + { + urls: 'turn:numb.viagenie.ca', + username: 'g.parant@thecodingmachine.com', + credential: 'itcugcOHxle9Acqi$' + }, + ] + } + }); + + //start listen signal for the peer connection + this.on('signal', (data: unknown) => { + this.sendWebrtcScreenSharingSignal(data); + }); + + this.on('stream', (stream: MediaStream) => { + this.stream(stream); + }); + + this.on('close', () => { + this.destroy(); + }); + + this.on('data', (chunk: Buffer) => { + // We unfortunately need to rely on an event to let the other party know a stream has stopped. + // It seems there is no native way to detect that. + const message = JSON.parse(chunk.toString('utf8')); + if (message.streamEnded !== true) { + console.error('Unexpected message on screen sharing peer connection'); + } + mediaManager.removeActiveScreenSharingVideo(this.userId); + }); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + this.on('error', (err: any) => { + console.error(`screen sharing error => ${this.userId} => ${err.code}`, err); + //mediaManager.isErrorScreenSharing(this.userId); + }); + + this.on('connect', () => { + // FIXME: we need to put the loader on the screen sharing connection + mediaManager.isConnected(this.userId); + console.info(`connect => ${this.userId}`); + }); + + this.pushScreenSharingToRemoteUser(); + } + + private sendWebrtcScreenSharingSignal(data: unknown) { + console.log("sendWebrtcScreenSharingSignal", data); + try { + this.connection.sendWebrtcScreenSharingSignal(data, this.userId); + }catch (e) { + console.error(`sendWebrtcScreenSharingSignal => ${this.userId}`, e); + } + } + + /** + * Sends received stream to screen. + */ + private stream(stream?: MediaStream) { + console.log(`ScreenSharingPeer::stream => ${this.userId}`, stream); + console.log(`stream => ${this.userId} => `, stream); + if(!stream){ + mediaManager.removeActiveScreenSharingVideo(this.userId); + this.isReceivingStream = false; + } else { + mediaManager.addStreamRemoteScreenSharing(this.userId, stream); + this.isReceivingStream = true; + } + } + + public isReceivingScreenSharingStream(): boolean { + return this.isReceivingStream; + } + + public destroy(error?: Error): void { + try { + mediaManager.removeActiveScreenSharingVideo(this.userId); + // FIXME: I don't understand why "Closing connection with" message is displayed TWICE before "Nb users in peerConnectionArray" + // I do understand the method closeConnection is called twice, but I don't understand how they manage to run in parallel. + //console.log('Closing connection with '+userId); + super.destroy(error); + //console.log('Nb users in peerConnectionArray '+this.PeerConnectionArray.size); + } catch (err) { + console.error("ScreenSharingPeer::destroy", err) + } + } + + private pushScreenSharingToRemoteUser() { + const localScreenCapture: MediaStream | null = mediaManager.localScreenCapture; + if(!localScreenCapture){ + return; + } + + this.addStream(localScreenCapture); + return; + } + + public stopPushingScreenSharingToRemoteUser(stream: MediaStream) { + this.removeStream(stream); + this.write(new Buffer(JSON.stringify({streamEnded: true}))); + } +} diff --git a/front/src/WebRtc/SimplePeer.ts b/front/src/WebRtc/SimplePeer.ts index fdc2d0c2..3acd65c5 100644 --- a/front/src/WebRtc/SimplePeer.ts +++ b/front/src/WebRtc/SimplePeer.ts @@ -1,21 +1,23 @@ import { Connection, WebRtcDisconnectMessageInterface, - WebRtcSignalMessageInterface, + WebRtcSignalReceivedMessageInterface, WebRtcStartMessageInterface } from "../Connection"; import { mediaManager } from "./MediaManager"; import * as SimplePeerNamespace from "simple-peer"; +import {ScreenSharingPeer} from "./ScreenSharingPeer"; +import {VideoPeer} from "./VideoPeer"; const Peer: SimplePeerNamespace.SimplePeer = require('simple-peer'); -export interface UserSimplePeer{ +export interface UserSimplePeerInterface{ userId: string; name?: string; initiator?: boolean; } export interface PeerConnectionListener { - onConnect(user: UserSimplePeer): void; + onConnect(user: UserSimplePeerInterface): void; onDisconnect(userId: string): void; } @@ -26,18 +28,25 @@ export interface PeerConnectionListener { export class SimplePeer { private Connection: Connection; private WebRtcRoomId: string; - private Users: Array = new Array(); + private Users: Array = new Array(); - private PeerConnectionArray: Map = new Map(); - private readonly updateLocalStreamCallback: (media: MediaStream) => void; + private PeerScreenSharingConnectionArray: Map = new Map(); + private PeerConnectionArray: Map = new Map(); + private readonly sendLocalVideoStreamCallback: (media: MediaStream) => void; + private readonly sendLocalScreenSharingStreamCallback: (media: MediaStream) => void; + private readonly stopLocalScreenSharingStreamCallback: (media: MediaStream) => void; private readonly peerConnectionListeners: Array = new Array(); constructor(Connection: Connection, WebRtcRoomId: string = "test-webrtc") { this.Connection = Connection; this.WebRtcRoomId = WebRtcRoomId; // We need to go through this weird bound function pointer in order to be able to "free" this reference later. - this.updateLocalStreamCallback = this.updatedLocalStream.bind(this); - mediaManager.onUpdateLocalStream(this.updateLocalStreamCallback); + this.sendLocalVideoStreamCallback = this.sendLocalVideoStream.bind(this); + this.sendLocalScreenSharingStreamCallback = this.sendLocalScreenSharingStream.bind(this); + this.stopLocalScreenSharingStreamCallback = this.stopLocalScreenSharingStream.bind(this); + mediaManager.onUpdateLocalStream(this.sendLocalVideoStreamCallback); + mediaManager.onStartScreenSharing(this.sendLocalScreenSharingStreamCallback); + mediaManager.onStopScreenSharing(this.stopLocalScreenSharingStreamCallback); this.initialise(); } @@ -55,11 +64,16 @@ export class SimplePeer { private initialise() { //receive signal by gemer - this.Connection.receiveWebrtcSignal((message: WebRtcSignalMessageInterface) => { + this.Connection.receiveWebrtcSignal((message: WebRtcSignalReceivedMessageInterface) => { this.receiveWebrtcSignal(message); }); - mediaManager.activeVisio(); + //receive signal by gemer + this.Connection.receiveWebrtcScreenSharingSignal((message: WebRtcSignalReceivedMessageInterface) => { + this.receiveWebrtcScreenSharingSignal(message); + }); + + mediaManager.showGameOverlay(); mediaManager.getCamera().then(() => { //receive message start @@ -79,7 +93,7 @@ export class SimplePeer { private receiveWebrtcStart(data: WebRtcStartMessageInterface) { this.WebRtcRoomId = data.roomId; this.Users = data.clients; - // Note: the clients array contain the list of all clients (event the ones we are already connected to in case a user joints a group) + // Note: the clients array contain the list of all clients (even the ones we are already connected to in case a user joints a group) // So we can receive a request we already had before. (which will abort at the first line of createPeerConnection) // TODO: refactor this to only send a message to connect to one user (rather than several users). // This would be symmetrical to the way we handle disconnection. @@ -93,7 +107,8 @@ export class SimplePeer { * server has two people connected, start the meet */ private startWebRtc() { - this.Users.forEach((user: UserSimplePeer) => { + console.warn('startWebRtc startWebRtc'); + this.Users.forEach((user: UserSimplePeerInterface) => { //if it's not an initiator, peer connection will be created when gamer will receive offer signal if(!user.initiator){ return; @@ -105,102 +120,63 @@ export class SimplePeer { /** * create peer connection to bind users */ - private createPeerConnection(user : UserSimplePeer) { - if(this.PeerConnectionArray.has(user.userId)) { - return; + private createPeerConnection(user : UserSimplePeerInterface) : VideoPeer | null{ + if( + this.PeerConnectionArray.has(user.userId) + ){ + return null; } - //console.log("Creating connection with peer "+user.userId); - let name = user.name; if(!name){ - const userSearch = this.Users.find((userSearch: UserSimplePeer) => userSearch.userId === user.userId); + const userSearch = this.Users.find((userSearch: UserSimplePeerInterface) => userSearch.userId === user.userId); if(userSearch) { name = userSearch.name; } } + mediaManager.removeActiveVideo(user.userId); mediaManager.addActiveVideo(user.userId, name); - const peer : SimplePeerNamespace.Instance = new Peer({ - initiator: user.initiator ? user.initiator : false, - reconnectTimer: 10000, - config: { - iceServers: [ - { - urls: 'stun:stun.l.google.com:19302' - }, - { - urls: 'turn:numb.viagenie.ca', - username: 'g.parant@thecodingmachine.com', - credential: 'itcugcOHxle9Acqi$' - }, - ] - }, + const peer = new VideoPeer(user.userId, user.initiator ? user.initiator : false, this.Connection); + // When a connection is established to a video stream, and if a screen sharing is taking place, + // the user sharing screen should also initiate a connection to the remote user! + peer.on('connect', () => { + if (mediaManager.localScreenCapture) { + this.sendLocalScreenSharingStreamToUser(user.userId); + } }); this.PeerConnectionArray.set(user.userId, peer); - //start listen signal for the peer connection - peer.on('signal', (data: unknown) => { - this.sendWebrtcSignal(data, user.userId); - }); - - peer.on('stream', (stream: MediaStream) => { - let videoActive = false; - let microphoneActive = false; - stream.getTracks().forEach((track : MediaStreamTrack) => { - if(track.kind === "audio"){ - microphoneActive = true; - } - if(track.kind === "video"){ - videoActive = true; - } - }); - if(microphoneActive){ - mediaManager.enabledMicrophoneByUserId(user.userId); - }else{ - mediaManager.disabledMicrophoneByUserId(user.userId); - } - - if(videoActive){ - mediaManager.enabledVideoByUserId(user.userId); - }else{ - mediaManager.disabledVideoByUserId(user.userId); - } - this.stream(user.userId, stream); - }); - - /*peer.on('track', (track: MediaStreamTrack, stream: MediaStream) => { - this.stream(user.userId, stream); - });*/ - - peer.on('close', () => { - this.closeConnection(user.userId); - }); - - // eslint-disable-next-line @typescript-eslint/no-explicit-any - peer.on('error', (err: any) => { - console.error(`error => ${user.userId} => ${err.code}`, err); - mediaManager.isError(user.userId); - }); - - peer.on('connect', () => { - mediaManager.isConnected(user.userId); - console.info(`connect => ${user.userId}`); - }); - - peer.on('data', (chunk: Buffer) => { - const data = JSON.parse(chunk.toString('utf8')); - if(data.type === "stream"){ - this.stream(user.userId, data.stream); - } - }); - - this.addMedia(user.userId); - for (const peerConnectionListener of this.peerConnectionListeners) { peerConnectionListener.onConnect(user); } + return peer; + } + + /** + * create peer connection to bind users + */ + private createPeerScreenSharingConnection(user : UserSimplePeerInterface) : ScreenSharingPeer | null{ + if( + this.PeerScreenSharingConnectionArray.has(user.userId) + ){ + return null; + } + + // We should display the screen sharing ONLY if we are not initiator + if (!user.initiator) { + mediaManager.removeActiveScreenSharingVideo(user.userId); + mediaManager.addScreenSharingActiveVideo(user.userId); + } + + const peer = new ScreenSharingPeer(user.userId, user.initiator ? user.initiator : false, this.Connection); + this.PeerScreenSharingConnectionArray.set(user.userId, peer); + + for (const peerConnectionListener of this.peerConnectionListeners) { + peerConnectionListener.onConnect(user); + } + return peer; } /** @@ -210,17 +186,18 @@ export class SimplePeer { */ private closeConnection(userId : string) { try { - mediaManager.removeActiveVideo(userId); + //mediaManager.removeActiveVideo(userId); const peer = this.PeerConnectionArray.get(userId); if (peer === undefined) { console.warn("Tried to close connection for user "+userId+" but could not find user") return; } + peer.destroy(); // FIXME: I don't understand why "Closing connection with" message is displayed TWICE before "Nb users in peerConnectionArray" // I do understand the method closeConnection is called twice, but I don't understand how they manage to run in parallel. //console.log('Closing connection with '+userId); - peer.destroy(); - this.PeerConnectionArray.delete(userId) + this.PeerConnectionArray.delete(userId); + this.closeScreenSharingConnection(userId); //console.log('Nb users in peerConnectionArray '+this.PeerConnectionArray.size); for (const peerConnectionListener of this.peerConnectionListeners) { peerConnectionListener.onDisconnect(userId); @@ -230,34 +207,49 @@ export class SimplePeer { } } + /** + * This is triggered twice. Once by the server, and once by a remote client disconnecting + * + * @param userId + */ + private closeScreenSharingConnection(userId : string) { + try { + mediaManager.removeActiveScreenSharingVideo(userId); + const peer = this.PeerScreenSharingConnectionArray.get(userId); + if (peer === undefined) { + console.warn("Tried to close connection for user "+userId+" but could not find user") + return; + } + // FIXME: I don't understand why "Closing connection with" message is displayed TWICE before "Nb users in peerConnectionArray" + // I do understand the method closeConnection is called twice, but I don't understand how they manage to run in parallel. + //console.log('Closing connection with '+userId); + peer.destroy(); + this.PeerScreenSharingConnectionArray.delete(userId) + //console.log('Nb users in peerConnectionArray '+this.PeerConnectionArray.size); + } catch (err) { + console.error("closeConnection", err) + } + } + public closeAllConnections() { for (const userId of this.PeerConnectionArray.keys()) { this.closeConnection(userId); } + + for (const userId of this.PeerScreenSharingConnectionArray.keys()) { + this.closeScreenSharingConnection(userId); + } } /** * Unregisters any held event handler. */ public unregister() { - mediaManager.removeUpdateLocalStreamEventListener(this.updateLocalStreamCallback); - } - - /** - * - * @param userId - * @param data - */ - private sendWebrtcSignal(data: unknown, userId : string) { - try { - this.Connection.sendWebrtcSignal(data, this.WebRtcRoomId, null, userId); - }catch (e) { - console.error(`sendWebrtcSignal => ${userId}`, e); - } + mediaManager.removeUpdateLocalStreamEventListener(this.sendLocalVideoStreamCallback); } // eslint-disable-next-line @typescript-eslint/no-explicit-any - private receiveWebrtcSignal(data: WebRtcSignalMessageInterface) { + private receiveWebrtcSignal(data: WebRtcSignalReceivedMessageInterface) { try { //if offer type, create peer connection if(data.signal.type === "offer"){ @@ -274,53 +266,126 @@ export class SimplePeer { } } - /** - * - * @param userId - * @param stream - */ - private stream(userId : string, stream: MediaStream) { - if(!stream){ - mediaManager.disabledVideoByUserId(userId); - mediaManager.disabledMicrophoneByUserId(userId); - return; + private receiveWebrtcScreenSharingSignal(data: WebRtcSignalReceivedMessageInterface) { + console.log("receiveWebrtcScreenSharingSignal", data); + try { + //if offer type, create peer connection + if(data.signal.type === "offer"){ + this.createPeerScreenSharingConnection(data); + } + const peer = this.PeerScreenSharingConnectionArray.get(data.userId); + if (peer !== undefined) { + peer.signal(data.signal); + } else { + console.error('Could not find peer whose ID is "'+data.userId+'" in receiveWebrtcScreenSharingSignal'); + } + } catch (e) { + console.error(`receiveWebrtcSignal => ${data.userId}`, e); } - mediaManager.addStreamRemoteVideo(userId, stream); } /** * * @param userId */ - private addMedia (userId : string) { + private pushVideoToRemoteUser(userId : string) { try { - const localStream: MediaStream|null = mediaManager.localStream; - const peer = this.PeerConnectionArray.get(userId); - if(localStream === null) { - //send fake signal - if(peer === undefined){ - return; - } - peer.write(new Buffer(JSON.stringify({ - type: "stream", - stream: null - }))); + const PeerConnection = this.PeerConnectionArray.get(userId); + if (!PeerConnection) { + throw new Error('While adding media, cannot find user with ID ' + userId); + } + const localStream: MediaStream | null = mediaManager.localStream; + PeerConnection.write(new Buffer(JSON.stringify(mediaManager.constraintsMedia))); + + if(!localStream){ return; } - if (peer === undefined) { - throw new Error('While adding media, cannot find user with ID '+userId); - } + for (const track of localStream.getTracks()) { - peer.addTrack(track, localStream); + PeerConnection.addTrack(track, localStream); } }catch (e) { - console.error(`addMedia => addMedia => ${userId}`, e); + console.error(`pushVideoToRemoteUser => ${userId}`, e); } } - updatedLocalStream(){ - this.Users.forEach((user: UserSimplePeer) => { - this.addMedia(user.userId); + private pushScreenSharingToRemoteUser(userId : string) { + const PeerConnection = this.PeerScreenSharingConnectionArray.get(userId); + if (!PeerConnection) { + throw new Error('While pushing screen sharing, cannot find user with ID ' + userId); + } + const localScreenCapture: MediaStream | null = mediaManager.localScreenCapture; + if(!localScreenCapture){ + return; + } + + for (const track of localScreenCapture.getTracks()) { + PeerConnection.addTrack(track, localScreenCapture); + } + return; + } + + public sendLocalVideoStream(){ + this.Users.forEach((user: UserSimplePeerInterface) => { + this.pushVideoToRemoteUser(user.userId); }) } + + /** + * Triggered locally when clicking on the screen sharing button + */ + public sendLocalScreenSharingStream() { + if (!mediaManager.localScreenCapture) { + console.error('Could not find localScreenCapture to share') + return; + } + + for (const user of this.Users) { + this.sendLocalScreenSharingStreamToUser(user.userId); + } + } + + /** + * Triggered locally when clicking on the screen sharing button + */ + public stopLocalScreenSharingStream(stream: MediaStream) { + for (const user of this.Users) { + this.stopLocalScreenSharingStreamToUser(user.userId, stream); + } + } + + private sendLocalScreenSharingStreamToUser(userId: string): void { + // If a connection already exists with user (because it is already sharing a screen with us... let's use this connection) + if (this.PeerScreenSharingConnectionArray.has(userId)) { + this.pushScreenSharingToRemoteUser(userId); + return; + } + + const screenSharingUser: UserSimplePeerInterface = { + userId, + initiator: true + }; + const PeerConnectionScreenSharing = this.createPeerScreenSharingConnection(screenSharingUser); + if (!PeerConnectionScreenSharing) { + return; + } + } + + private stopLocalScreenSharingStreamToUser(userId: string, stream: MediaStream): void { + const PeerConnectionScreenSharing = this.PeerScreenSharingConnectionArray.get(userId); + if (!PeerConnectionScreenSharing) { + throw new Error('Weird, screen sharing connection to user ' + userId + 'not found') + } + + console.log("updatedScreenSharing => destroy", PeerConnectionScreenSharing); + + // Stop sending stream and close peer connection if peer is not sending stream too + PeerConnectionScreenSharing.stopPushingScreenSharingToRemoteUser(stream); + + if (!PeerConnectionScreenSharing.isReceivingScreenSharingStream()) { + PeerConnectionScreenSharing.destroy(); + + this.PeerScreenSharingConnectionArray.delete(userId); + } + } } diff --git a/front/src/WebRtc/VideoPeer.ts b/front/src/WebRtc/VideoPeer.ts new file mode 100644 index 00000000..ec7f2576 --- /dev/null +++ b/front/src/WebRtc/VideoPeer.ts @@ -0,0 +1,128 @@ +import * as SimplePeerNamespace from "simple-peer"; +import {mediaManager} from "./MediaManager"; +import {Connection} from "../Connection"; + +const Peer: SimplePeerNamespace.SimplePeer = require('simple-peer'); + +/** + * A peer connection used to transmit video / audio signals between 2 peers. + */ +export class VideoPeer extends Peer { + constructor(private userId: string, initiator: boolean, private connection: Connection) { + super({ + initiator: initiator ? initiator : false, + reconnectTimer: 10000, + config: { + iceServers: [ + { + urls: 'stun:stun.l.google.com:19302' + }, + { + urls: 'turn:numb.viagenie.ca', + username: 'g.parant@thecodingmachine.com', + credential: 'itcugcOHxle9Acqi$' + }, + ] + } + }); + + //start listen signal for the peer connection + this.on('signal', (data: unknown) => { + this.sendWebrtcSignal(data); + }); + + this.on('stream', (stream: MediaStream) => { + this.stream(stream); + }); + + /*peer.on('track', (track: MediaStreamTrack, stream: MediaStream) => { + });*/ + + this.on('close', () => { + this.destroy(); + }); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + this.on('error', (err: any) => { + console.error(`error => ${this.userId} => ${err.code}`, err); + mediaManager.isError(userId); + }); + + this.on('connect', () => { + mediaManager.isConnected(this.userId); + console.info(`connect => ${this.userId}`); + }); + + this.on('data', (chunk: Buffer) => { + const constraint = JSON.parse(chunk.toString('utf8')); + console.log("data", constraint); + if (constraint.audio) { + mediaManager.enabledMicrophoneByUserId(this.userId); + } else { + mediaManager.disabledMicrophoneByUserId(this.userId); + } + + if (constraint.video || constraint.screen) { + mediaManager.enabledVideoByUserId(this.userId); + } else { + this.stream(undefined); + mediaManager.disabledVideoByUserId(this.userId); + } + }); + + this.pushVideoToRemoteUser(); + } + + private sendWebrtcSignal(data: unknown) { + try { + this.connection.sendWebrtcSignal(data, this.userId); + }catch (e) { + console.error(`sendWebrtcSignal => ${this.userId}`, e); + } + } + + /** + * Sends received stream to screen. + */ + private stream(stream?: MediaStream) { + console.log(`VideoPeer::stream => ${this.userId}`, stream); + if(!stream){ + mediaManager.disabledVideoByUserId(this.userId); + mediaManager.disabledMicrophoneByUserId(this.userId); + } else { + mediaManager.addStreamRemoteVideo(this.userId, stream); + } + } + + /** + * This is triggered twice. Once by the server, and once by a remote client disconnecting + */ + public destroy(error?: Error): void { + try { + mediaManager.removeActiveVideo(this.userId); + // FIXME: I don't understand why "Closing connection with" message is displayed TWICE before "Nb users in peerConnectionArray" + // I do understand the method closeConnection is called twice, but I don't understand how they manage to run in parallel. + //console.log('Closing connection with '+userId); + super.destroy(error); + } catch (err) { + console.error("VideoPeer::destroy", err) + } + } + + private pushVideoToRemoteUser() { + try { + const localStream: MediaStream | null = mediaManager.localStream; + this.write(new Buffer(JSON.stringify(mediaManager.constraintsMedia))); + + if(!localStream){ + return; + } + + for (const track of localStream.getTracks()) { + this.addTrack(track, localStream); + } + }catch (e) { + console.error(`pushVideoToRemoteUser => ${this.userId}`, e); + } + } +} diff --git a/front/tsconfig.json b/front/tsconfig.json index e56a6ee7..64d71e42 100644 --- a/front/tsconfig.json +++ b/front/tsconfig.json @@ -3,9 +3,8 @@ "outDir": "./dist/", "sourceMap": true, "moduleResolution": "node", - "noImplicitAny": true, "module": "CommonJS", - "target": "es5", + "target": "es6", "downlevelIteration": true, "jsx": "react", "allowJs": true,