From fdb40ec3e21a503ec3c5fca198c404b1af57b2dc Mon Sep 17 00:00:00 2001 From: gparant Date: Sat, 2 May 2020 20:46:02 +0200 Subject: [PATCH] Fix webrtc multi --- back/src/Controller/IoSocketController.ts | 73 ++++++-- front/dist/resources/style/style.css | 1 - front/src/Connexion.ts | 14 +- front/src/Phaser/Login/LogincScene.ts | 2 + front/src/WebRtc/MediaManager.ts | 14 +- front/src/WebRtc/SimplePeer.ts | 208 ++++++++++++---------- 6 files changed, 195 insertions(+), 117 deletions(-) diff --git a/back/src/Controller/IoSocketController.ts b/back/src/Controller/IoSocketController.ts index 967c78a5..96fdb212 100644 --- a/back/src/Controller/IoSocketController.ts +++ b/back/src/Controller/IoSocketController.ts @@ -12,11 +12,13 @@ import {Group} from "_Model/Group"; enum SockerIoEvent { CONNECTION = "connection", - DISCONNECTION = "disconnect", + DISCONNECT = "disconnect", JOIN_ROOM = "join-room", USER_POSITION = "user-position", WEBRTC_SIGNAL = "webrtc-signal", + WEBRTC_OFFER = "webrtc-offer", WEBRTC_START = "webrtc-start", + WEBRTC_DISCONNECT = "webrtc-disconect", MESSAGE_ERROR = "message-error", } @@ -77,9 +79,7 @@ export class IoSocketController{ this.saveUserInformation((socket as ExSocketInterface), messageUserPosition); //add function to refresh position user in real time. - let rooms = (this.Io.sockets.adapter.rooms as ExtRoomsInterface); - rooms.refreshUserPosition = RefreshUserPositionFunction; - rooms.refreshUserPosition(rooms, this.Io); + this.refreshUserPosition(); socket.to(messageUserPosition.roomId).emit(SockerIoEvent.JOIN_ROOM, messageUserPosition.toString()); }); @@ -97,30 +97,41 @@ export class IoSocketController{ this.saveUserInformation((socket as ExSocketInterface), messageUserPosition); //refresh position of all user in all rooms in real time - let rooms = (this.Io.sockets.adapter.rooms as ExtRoomsInterface); - if(!rooms.refreshUserPosition){ - rooms.refreshUserPosition = RefreshUserPositionFunction; - } - rooms.refreshUserPosition(rooms, this.Io); + this.refreshUserPosition(); }); socket.on(SockerIoEvent.WEBRTC_SIGNAL, (message : string) => { let data : any = JSON.parse(message); - //send only at user - let clients: Array = Object.values(this.Io.sockets.sockets); - for(let i = 0; i < clients.length; i++){ - let client : ExSocketInterface = clients[i]; - if(client.userId !== data.receiverId){ - continue - } - client.emit(SockerIoEvent.WEBRTC_SIGNAL, message); - break; + let client = this.searchClientById(data.receiverId); + if(!client){ + console.error("client doesn't exist for ", data.receiverId); + return; } + return client.emit(SockerIoEvent.WEBRTC_SIGNAL, message); }); - socket.on(SockerIoEvent.DISCONNECTION, (reason : string) => { + socket.on(SockerIoEvent.WEBRTC_OFFER, (message : string) => { + let data : any = JSON.parse(message); + + //send only at user + let client = this.searchClientById(data.receiverId); + if(!client){ + console.error("client doesn't exist for ", data.receiverId); + return; + } + client.emit(SockerIoEvent.WEBRTC_OFFER, message); + }); + + socket.on(SockerIoEvent.DISCONNECT, () => { let Client = (socket as ExSocketInterface); + socket.broadcast.emit(SockerIoEvent.WEBRTC_DISCONNECT, JSON.stringify({ + userId: Client.userId + })); + + //refresh position of all user in all rooms in real time + this.refreshUserPosition(); + //leave group of user this.World.leave(Client); @@ -138,6 +149,21 @@ export class IoSocketController{ }); } + /** + * + * @param userId + */ + searchClientById(userId : string) : ExSocketInterface | null{ + let clients: Array = Object.values(this.Io.sockets.sockets); + for(let i = 0; i < clients.length; i++){ + let client : ExSocketInterface = clients[i]; + if(client.userId !== userId){ + continue + } + return client; + } + return null; + } /** * * @param socket @@ -181,6 +207,15 @@ export class IoSocketController{ socket.userId = message.userId; } + refreshUserPosition(){ + //refresh position of all user in all rooms in real time + let rooms = (this.Io.sockets.adapter.rooms as ExtRoomsInterface); + if(!rooms.refreshUserPosition){ + rooms.refreshUserPosition = RefreshUserPositionFunction; + } + rooms.refreshUserPosition(rooms, this.Io); + } + //Hydrate and manage error hydrateMessageReceive(message : string) : MessageUserPosition | Error{ try { diff --git a/front/dist/resources/style/style.css b/front/dist/resources/style/style.css index 58965b2e..049c3d1d 100644 --- a/front/dist/resources/style/style.css +++ b/front/dist/resources/style/style.css @@ -16,7 +16,6 @@ top: 10px; right: 10px; margin: 5px; - background-color: white; } .activeCam video#myCamVideo{ width: 200px; diff --git a/front/src/Connexion.ts b/front/src/Connexion.ts index dc48833c..56531b67 100644 --- a/front/src/Connexion.ts +++ b/front/src/Connexion.ts @@ -7,9 +7,11 @@ import {API_URL, ROOM} from "./Enum/EnvironmentVariable"; enum EventMessage{ WEBRTC_SIGNAL = "webrtc-signal", WEBRTC_START = "webrtc-start", + WEBRTC_JOIN_ROOM = "webrtc-join-room", JOIN_ROOM = "join-room", USER_POSITION = "user-position", - MESSAGE_ERROR = "message-error" + MESSAGE_ERROR = "message-error", + WEBRTC_DISCONNECT = "webrtc-disconect" } class Message { @@ -131,6 +133,8 @@ export interface ConnexionInterface { receiveWebrtcSignal(callBack: Function): void; receiveWebrtcStart(callBack: Function): void; + + disconnectMessage(callBack: Function): void; } export class Connexion implements ConnexionInterface { @@ -227,7 +231,7 @@ export class Connexion implements ConnexionInterface { } sendWebrtcSignal(signal: any, roomId: string, userId? : string, receiverId? : string) { - this.socket.emit(EventMessage.WEBRTC_SIGNAL, JSON.stringify({ + return this.socket.emit(EventMessage.WEBRTC_SIGNAL, JSON.stringify({ userId: userId ? userId : this.userId, receiverId: receiverId ? receiverId : this.userId, roomId: roomId, @@ -240,7 +244,7 @@ export class Connexion implements ConnexionInterface { } receiveWebrtcSignal(callback: Function) { - this.socket.on(EventMessage.WEBRTC_SIGNAL, callback); + return this.socket.on(EventMessage.WEBRTC_SIGNAL, callback); } errorMessage(): void { @@ -248,4 +252,8 @@ export class Connexion implements ConnexionInterface { console.error(EventMessage.MESSAGE_ERROR, message); }) } + + disconnectMessage(callback: Function): void { + this.socket.on(EventMessage.WEBRTC_DISCONNECT, callback); + } } \ No newline at end of file diff --git a/front/src/Phaser/Login/LogincScene.ts b/front/src/Phaser/Login/LogincScene.ts index 15ead519..c9c22f0f 100644 --- a/front/src/Phaser/Login/LogincScene.ts +++ b/front/src/Phaser/Login/LogincScene.ts @@ -3,6 +3,8 @@ import {TextField} from "../Components/TextField"; import {TextInput} from "../Components/TextInput"; import {ClickButton} from "../Components/ClickButton"; import {GameSceneName} from "../Game/GameScene"; +import {SimplePeer} from "../../WebRtc/SimplePeer"; +import {Connexion} from "../../Connexion"; //todo: put this constants in a dedicated file export const LoginSceneName = "LoginScene"; diff --git a/front/src/WebRtc/MediaManager.ts b/front/src/WebRtc/MediaManager.ts index 5b8d0076..4a875af2 100644 --- a/front/src/WebRtc/MediaManager.ts +++ b/front/src/WebRtc/MediaManager.ts @@ -6,7 +6,7 @@ export class MediaManager { cinema: any = null; microphoneClose: any = null; microphone: any = null; - constraintsMedia = {audio: false, video: true}; + constraintsMedia = {audio: true, video: true}; getCameraPromise : Promise = null; constructor() { @@ -37,9 +37,6 @@ export class MediaManager { this.disabledCamera(); //update tracking }); - - this.enabledCamera(); - this.enabledMicrophone(); } activeVisio(){ @@ -127,6 +124,15 @@ export class MediaManager { this.remoteVideo[(userId as any)] = document.getElementById(userId); } + /** + * + * @param userId + * @param stream + */ + addStreamRemoteVideo(userId : string, stream : MediaStream){ + this.remoteVideo[(userId as any)].srcObject = stream; + } + /** * * @param userId diff --git a/front/src/WebRtc/SimplePeer.ts b/front/src/WebRtc/SimplePeer.ts index 776ed470..0e430326 100644 --- a/front/src/WebRtc/SimplePeer.ts +++ b/front/src/WebRtc/SimplePeer.ts @@ -4,67 +4,52 @@ let Peer = require('simple-peer'); export interface SimplePeerInterface { } -enum PeerConnexionStatus{ - DISABLED = 1, - ACTIVATED = 2 -} + export class SimplePeer { private Connexion: ConnexionInterface; - private MediaManager: MediaManager; private WebRtcRoomId: string; private Users: Array; - private PeerConnexionArray: Array = new Array(); + private MediaManager: MediaManager; - private PeerConnexionStatus : number = PeerConnexionStatus.DISABLED; + private PeerConnexionArray: Array = new Array(); constructor(Connexion: ConnexionInterface, WebRtcRoomId: string = "test-webrtc") { this.Connexion = Connexion; this.WebRtcRoomId = WebRtcRoomId; this.MediaManager = new MediaManager(); + this.PeerConnexionArray = new Array(); + this.initialise(); } /** * permit to listen when user could start visio */ - private initialise(){ - - //receive message start - this.Connexion.receiveWebrtcStart((message: string) => { - this.receiveWebrtcStart(message); - }); - - //when button to call is clicked, start video - /*this.MediaManager.getElementActivePhone().addEventListener("click", () => { - this.startWebRtc(); - this.disablePhone(); - });*/ - - return this.MediaManager.getCamera().then((stream: MediaStream) => { - this.MediaManager.activeVisio(); - this.MediaManager.localStream = stream; - }); - } - /** - * server has two person connected, start the meet - */ - private startWebRtc() { - //create pear connexion - this.createPeerConnexion(); + private initialise() { //receive signal by gemer this.Connexion.receiveWebrtcSignal((message: string) => { this.receiveWebrtcSignal(message); }); - // add media or new media for all peer connexion - this.Users.forEach((user: any) => { - this.addMedia(user.userId); + this.MediaManager.activeVisio(); + this.MediaManager.getCamera().then(() => { + + //receive message start + this.Connexion.receiveWebrtcStart((message: string) => { + this.receiveWebrtcStart(message); + }); + + }).catch((err) => { + console.error("err", err); }); - //change status to manage other user - this.PeerConnexionStatus = PeerConnexionStatus.ACTIVATED; + //receive signal by gemer + this.Connexion.disconnectMessage((message: string) => { + let data = JSON.parse(message); + this.closeConnexion(data.userId); + }); } /** @@ -76,51 +61,92 @@ export class SimplePeer { this.WebRtcRoomId = data.roomId; this.Users = data.clients; - console.log("receiveWebrtcStart", this.Users); - //start connexion this.startWebRtc(); } - - private createPeerConnexion() { + /** + * server has two person connected, start the meet + */ + private startWebRtc() { this.Users.forEach((user: any) => { - if (this.PeerConnexionArray[user.userId]) { + //if it's not an initiator, peer connexion will be created when gamer will receive offer signal + if(!user.initiator){ return; } - this.MediaManager.addActiveVideo(user.userId); - - console.info("createPeerConnexion => create peerConexion", user); - this.PeerConnexionArray[user.userId] = new Peer({ - initiator: user.initiator, - config: { iceServers: [{ urls: 'stun:stun.l.google.com:19302' }, { urls: 'stun:global.stun.twilio.com:3478?transport=udp' }] } - }); - - //add lof info PeerConnexionArray - //this.PeerConnexionArray[user.userId]._debug = console.info; - - this.PeerConnexionArray[user.userId].on('signal', (data: any) => { - console.info("createPeerConnexion => sendWebrtcSignal : "+user.userId, data); - this.sendWebrtcSignal(data, user.userId); - }); - - this.PeerConnexionArray[user.userId].on('stream', (stream: MediaStream) => { - this.stream(user.userId, stream); - }); - - this.PeerConnexionArray[user.userId].on('close', () => { - console.info("createPeerConnexion => close", user.userId); - this.closeConnexion(user.userId); - }); - - this.addMedia(user.userId); + this.createPeerConnexion(user); }); } - private closeConnexion(userId : string){ - // @ts-ignore - this.PeerConnexionArray[userId] = null; - this.MediaManager.removeActiveVideo(userId) + /** + * create peer connexion to bind users + */ + private createPeerConnexion(user : any) { + if(this.PeerConnexionArray[user.userId]) { + return; + } + + this.MediaManager.removeActiveVideo(user.userId); + this.MediaManager.addActiveVideo(user.userId); + + this.PeerConnexionArray[user.userId] = new Peer({ + initiator: user.initiator ? user.initiator : false, + reconnectTimer: 10000, + config: { + iceServers: [ + { + urls: 'stun:stun.l.google.com:19302' + }, + { + urls: 'turn:numb.viagenie.ca', + username: 'g.parant@thecodingmachine.com', + credential: 'Muzuvo$6' + }, + ] + }, + }); + + //start listen signal for the peer connexion + this.PeerConnexionArray[user.userId].on('signal', (data: any) => { + this.sendWebrtcSignal(data, user.userId); + }); + + this.PeerConnexionArray[user.userId].on('stream', (stream: MediaStream) => { + this.stream(user.userId, stream); + }); + + this.PeerConnexionArray[user.userId].on('track', (track: MediaStreamTrack, stream: MediaStream) => { + this.stream(user.userId, stream); + }); + + this.PeerConnexionArray[user.userId].on('close', () => { + this.closeConnexion(user.userId); + }); + + this.PeerConnexionArray[user.userId].on('error', (err: any) => { + console.error(`error => ${user.userId} => ${err.code}`, err); + }); + + this.PeerConnexionArray[user.userId].on('connect', () => { + console.info(`connect => ${user.userId}`); + }); + + this.addMedia(user.userId); + } + + private closeConnexion(userId : string) { + try { + this.MediaManager.removeActiveVideo(userId); + if (!this.PeerConnexionArray[(userId as any)]) { + return; + } + // @ts-ignore + this.PeerConnexionArray[(userId as any)].destroy(); + this.PeerConnexionArray[(userId as any)] = null; + delete this.PeerConnexionArray[(userId as any)]; + } catch (err) { + console.error("closeConnexion", err) + } } /** @@ -129,7 +155,11 @@ export class SimplePeer { * @param data */ private sendWebrtcSignal(data: any, userId : string) { - this.Connexion.sendWebrtcSignal(data, this.WebRtcRoomId, null, userId); + try { + this.Connexion.sendWebrtcSignal(data, this.WebRtcRoomId, null, userId); + }catch (e) { + console.error(`sendWebrtcSignal => ${userId}`, e); + } } /** @@ -138,12 +168,15 @@ export class SimplePeer { */ private receiveWebrtcSignal(message: string) { let data = JSON.parse(message); - console.log("receiveWebrtcSignal", data.userId); - console.log("receiveWebrtcSignal => data", data); - if(!this.PeerConnexionArray[data.userId]){ - return; + try { + //if offer type, create peer connexion + if(data.signal.type === "offer"){ + this.createPeerConnexion(data); + } + this.PeerConnexionArray[data.userId].signal(data.signal); + } catch (e) { + console.error(`receiveWebrtcSignal => ${data.userId}`, e); } - this.PeerConnexionArray[data.userId].signal(data.signal); } /** @@ -152,7 +185,7 @@ export class SimplePeer { * @param stream */ private stream(userId : any, stream: MediaStream) { - this.MediaManager.remoteVideo[userId].srcObject = stream; + this.MediaManager.addStreamRemoteVideo(userId, stream); } /** @@ -160,18 +193,13 @@ export class SimplePeer { * @param userId */ private addMedia (userId : any = null) { - if (!this.MediaManager.localStream || !this.PeerConnexionArray[userId]) { - return; + try { + let transceiver : any = null; + this.MediaManager.localStream.getTracks().forEach( + transceiver = (track: MediaStreamTrack) => this.PeerConnexionArray[userId].addTrack(track, this.MediaManager.localStream) + ) + }catch (e) { + console.error(`addMedia => addMedia => ${userId}`, e); } - this.PeerConnexionArray[userId].addStream(this.MediaManager.localStream) // <- add streams to peer dynamically - return; - } - - private activePhone(){ - this.MediaManager.activePhoneOpen(); - } - - private disablePhone(){ - this.MediaManager.disablePhoneOpen(); } } \ No newline at end of file