Adding a scene to configure the webcam

front/dist/index.html

@@ -59,6 +59,10 @@
                 </div>-->
             </div>
         </div>
+        <div id="webRtcSetup" class="webrtcsetup">
+                <img id="webRtcSetupNoVideo" class="background-img" src="resources/logos/cinema-close.svg">
+                <video id="myCamVideoSetup" autoplay muted></video>
+        </div>
         <audio id="audio-webrtc-in">
             <source src="/resources/objects/webrtc-in.mp3" type="audio/mp3">
         </audio>

front/dist/resources/style/style.css

@@ -207,3 +207,29 @@ video{
         opacity: 0;
     }
 }
+
+.webrtcsetup{
+    display: none;
+    position: absolute;
+    top: 230px;
+    left: 0;
+    right: 0;
+    margin-left: auto;
+    margin-right: auto;
+    height: 50%;
+    width: 50%;
+}
+.webrtcsetup .background-img {
+    position: relative;
+    display: block;
+    width: 40%;
+    height: 60%;
+    margin-left: auto;
+    margin-right: auto;
+}
+#myCamVideoSetup {
+    width: 100%;
+}
+.webrtcsetup.active{
+    display: block;
+}

front/src/Phaser/Components/SoundMeter.ts

@@ -0,0 +1,138 @@
+/**
+ * Class to measure the sound volume of a media stream
+ */
+export class SoundMeter {
+    private instant: number;
+    private clip: number;
+    //private script: ScriptProcessorNode;
+    private analyser: AnalyserNode|undefined;
+    private dataArray: Uint8Array|undefined;
+    private context: AudioContext|undefined;
+    private source: MediaStreamAudioSourceNode|undefined;
+
+    constructor() {
+        this.instant = 0.0;
+        this.clip = 0.0;
+        //this.script = context.createScriptProcessor(2048, 1, 1);
+    }
+
+    private init(context: AudioContext) {
+        if (this.context === undefined) {
+            this.context = context;
+            this.analyser = this.context.createAnalyser();
+
+            this.analyser.fftSize = 2048;
+            const bufferLength = this.analyser.fftSize;
+            this.dataArray = new Uint8Array(bufferLength);
+        }
+    }
+
+    public connectToSource(stream: MediaStream, context: AudioContext): void
+    {
+        this.init(context);
+
+        this.source = this.context?.createMediaStreamSource(stream);
+        if (this.analyser !== undefined) {
+            this.source?.connect(this.analyser);
+        }
+        //analyser.connect(distortion);
+        //distortion.connect(this.context.destination);
+        //this.analyser.connect(this.context.destination);
+
+
+    }
+
+    public getVolume(): number {
+        if (this.context === undefined || this.dataArray === undefined || this.analyser === undefined) {
+            return 0;
+        }
+        this.analyser.getByteFrequencyData(this.dataArray);
+
+
+        const input = this.dataArray;
+        let i;
+        let sum = 0.0;
+        //let clipcount = 0;
+        for (i = 0; i < input.length; ++i) {
+            sum += input[i] * input[i];
+        //    if (Math.abs(input[i]) > 0.99) {
+        //        clipcount += 1;
+        //    }
+        }
+        this.instant = Math.sqrt(sum / input.length);
+        //this.slow = 0.95 * that.slow + 0.05 * that.instant;
+        //this.clip = clipcount / input.length;
+
+        //console.log('instant', this.instant, 'clip', this.clip);
+
+        return this.instant;
+    }
+
+    public stop(): void {
+        if (this.context === undefined) {
+            return;
+        }
+        if (this.source !== undefined) {
+            this.source.disconnect();
+        }
+        this.context = undefined;
+        this.analyser = undefined;
+        this.dataArray = undefined;
+        this.source = undefined;
+    }
+
+}
+
+
+// Meter class that generates a number correlated to audio volume.
+// The meter class itself displays nothing, but it makes the
+// instantaneous and time-decaying volumes available for inspection.
+// It also reports on the fraction of samples that were at or near
+// the top of the measurement range.
+/*function SoundMeter(context) {
+    this.context = context;
+    this.instant = 0.0;
+    this.slow = 0.0;
+    this.clip = 0.0;
+    this.script = context.createScriptProcessor(2048, 1, 1);
+    const that = this;
+    this.script.onaudioprocess = function(event) {
+        const input = event.inputBuffer.getChannelData(0);
+        let i;
+        let sum = 0.0;
+        let clipcount = 0;
+        for (i = 0; i < input.length; ++i) {
+            sum += input[i] * input[i];
+            if (Math.abs(input[i]) > 0.99) {
+                clipcount += 1;
+            }
+        }
+        that.instant = Math.sqrt(sum / input.length);
+        that.slow = 0.95 * that.slow + 0.05 * that.instant;
+        that.clip = clipcount / input.length;
+    };
+}
+
+SoundMeter.prototype.connectToSource = function(stream, callback) {
+    console.log('SoundMeter connecting');
+    try {
+        this.mic = this.context.createMediaStreamSource(stream);
+        this.mic.connect(this.script);
+        // necessary to make sample run, but should not be.
+        this.script.connect(this.context.destination);
+        if (typeof callback !== 'undefined') {
+            callback(null);
+        }
+    } catch (e) {
+        console.error(e);
+        if (typeof callback !== 'undefined') {
+            callback(e);
+        }
+    }
+};
+
+SoundMeter.prototype.stop = function() {
+    this.mic.disconnect();
+    this.script.disconnect();
+};
+*/

front/src/Phaser/Components/SoundMeterSprite.ts

@@ -0,0 +1,44 @@
+import Container = Phaser.GameObjects.Container;
+import {Scene} from "phaser";
+import GameObject = Phaser.GameObjects.GameObject;
+import Rectangle = Phaser.GameObjects.Rectangle;
+
+
+export class SoundMeterSprite extends Container {
+    private rectangles: Rectangle[] = new Array<Rectangle>();
+    private static readonly NB_BARS = 20;
+
+    constructor(scene: Scene, x?: number, y?: number, children?: GameObject[]) {
+        super(scene, x, y, children);
+
+        for (let i = 0; i < SoundMeterSprite.NB_BARS; i++) {
+            const rectangle = new Rectangle(scene, i * 13, 0, 10, 20, (Math.round(255 - i * 255 / SoundMeterSprite.NB_BARS) << 8) + (Math.round(i * 255 / SoundMeterSprite.NB_BARS) << 16));
+            this.add(rectangle);
+            this.rectangles.push(rectangle);
+        }
+    }
+
+    /**
+     * A number between 0 and 100
+     *
+     * @param volume
+     */
+    public setVolume(volume: number): void {
+
+        const normalizedVolume = volume / 100 * SoundMeterSprite.NB_BARS;
+        for (let i = 0; i < SoundMeterSprite.NB_BARS; i++) {
+            if (normalizedVolume < i) {
+                this.rectangles[i].alpha = 0.5;
+            } else {
+                this.rectangles[i].alpha = 1;
+            }
+        }
+    }
+
+    public getWidth(): number {
+        return SoundMeterSprite.NB_BARS * 13;
+    }
+
+
+
+}

front/src/Phaser/Login/EnableCameraScene.ts

@@ -0,0 +1,294 @@
+import {gameManager} from "../Game/GameManager";
+import {TextField} from "../Components/TextField";
+import {ClickButton} from "../Components/ClickButton";
+import Image = Phaser.GameObjects.Image;
+import Rectangle = Phaser.GameObjects.Rectangle;
+import {PLAYER_RESOURCES, PlayerResourceDescriptionInterface} from "../Entity/Character";
+import {GameSceneInitInterface} from "../Game/GameScene";
+import {StartMapInterface} from "../../Connection";
+import {mediaManager, MediaManager} from "../../WebRtc/MediaManager";
+import {RESOLUTION} from "../../Enum/EnvironmentVariable";
+import {SoundMeter} from "../Components/SoundMeter";
+import {SoundMeterSprite} from "../Components/SoundMeterSprite";
+
+export const EnableCameraSceneName = "EnableCameraScene";
+enum LoginTextures {
+    playButton = "play_button",
+    icon = "icon",
+    mainFont = "main_font",
+    arrowRight = "arrow_right",
+    arrowUp = "arrow_up"
+}
+
+export class EnableCameraScene extends Phaser.Scene {
+    private textField: TextField;
+    private pressReturnField: TextField;
+    private cameraNameField: TextField;
+    private logo: Image;
+    private arrowLeft: Image;
+    private arrowRight: Image;
+    private arrowDown: Image;
+    private arrowUp: Image;
+    private microphonesList: InputDeviceInfo[] = new Array<InputDeviceInfo>();
+    private camerasList: InputDeviceInfo[] = new Array<InputDeviceInfo>();
+    private cameraSelected: number = 0;
+    private microphoneSelected: number = 0;
+    private soundMeter: SoundMeter;
+    private soundMeterSprite: SoundMeterSprite;
+    private microphoneNameField: TextField;
+
+    constructor() {
+        super({
+            key: EnableCameraSceneName
+        });
+        this.soundMeter = new SoundMeter();
+    }
+
+    preload() {
+        this.load.image(LoginTextures.playButton, "resources/objects/play_button.png");
+        this.load.image(LoginTextures.icon, "resources/logos/tcm_full.png");
+        this.load.image(LoginTextures.arrowRight, "resources/objects/arrow_right.png");
+        this.load.image(LoginTextures.arrowUp, "resources/objects/arrow_up.png");
+        // Note: arcade.png from the Phaser 3 examples at: https://github.com/photonstorm/phaser3-examples/tree/master/public/assets/fonts/bitmap
+        this.load.bitmapFont(LoginTextures.mainFont, 'resources/fonts/arcade.png', 'resources/fonts/arcade.xml');
+    }
+
+    create() {
+        this.textField = new TextField(this, this.game.renderer.width / 2, 50, 'Turn on your camera and microphone');
+        this.textField.setOrigin(0.5).setCenterAlign();
+
+        this.pressReturnField = new TextField(this, this.game.renderer.width / 2, this.game.renderer.height - 30, 'Press enter to start');
+        this.pressReturnField.setOrigin(0.5).setCenterAlign();
+
+        this.cameraNameField = new TextField(this, this.game.renderer.width / 2, this.game.renderer.height - 60, '');
+        this.cameraNameField.setOrigin(0.5).setCenterAlign();
+
+        this.microphoneNameField = new TextField(this, this.game.renderer.width / 2, this.game.renderer.height - 40, '');
+        this.microphoneNameField.setOrigin(0.5).setCenterAlign();
+
+        this.arrowRight = new Image(this, 0, 0, LoginTextures.arrowRight);
+        this.arrowRight.setOrigin(0.5, 0.5);
+        this.arrowRight.setVisible(false);
+        this.add.existing(this.arrowRight);
+
+        this.arrowLeft = new Image(this, 0, 0, LoginTextures.arrowRight);
+        this.arrowLeft.setOrigin(0.5, 0.5);
+        this.arrowLeft.setVisible(false);
+        this.arrowLeft.flipX = true;
+        this.add.existing(this.arrowLeft);
+
+        this.arrowUp = new Image(this, 0, 0, LoginTextures.arrowUp);
+        this.arrowUp.setOrigin(0.5, 0.5);
+        this.arrowUp.setVisible(false);
+        this.add.existing(this.arrowUp);
+
+        this.arrowDown = new Image(this, 0, 0, LoginTextures.arrowUp);
+        this.arrowDown.setOrigin(0.5, 0.5);
+        this.arrowDown.setVisible(false);
+        this.arrowDown.flipY = true;
+        this.add.existing(this.arrowDown);
+
+        this.logo = new Image(this, this.game.renderer.width - 30, this.game.renderer.height - 20, LoginTextures.icon);
+        this.add.existing(this.logo);
+
+        this.input.keyboard.on('keyup-ENTER', () => {
+            return this.login();
+        });
+
+        this.getElementByIdOrFail<HTMLDivElement>('webRtcSetup').classList.add('active');
+
+        const mediaPromise = mediaManager.getCamera();
+        mediaPromise.then(this.getDevices.bind(this));
+        mediaPromise.then(this.setupStream.bind(this));
+
+        this.input.keyboard.on('keydown-RIGHT', () => {
+            if (this.cameraSelected === this.camerasList.length - 1) {
+                return;
+            }
+            this.cameraSelected++;
+            // TODO: the change of camera should be OBSERVED (reactive)
+            mediaManager.setCamera(this.camerasList[this.cameraSelected].deviceId).then(this.setupStream.bind(this));
+        });
+        this.input.keyboard.on('keydown-LEFT', () => {
+            if (this.cameraSelected === 0) {
+                return;
+            }
+            this.cameraSelected--;
+            mediaManager.setCamera(this.camerasList[this.cameraSelected].deviceId).then(this.setupStream.bind(this));
+        });
+        this.input.keyboard.on('keydown-DOWN', () => {
+            if (this.microphoneSelected === this.microphonesList.length - 1) {
+                return;
+            }
+            this.microphoneSelected++;
+            // TODO: the change of camera should be OBSERVED (reactive)
+            mediaManager.setMicrophone(this.microphonesList[this.microphoneSelected].deviceId).then(this.setupStream.bind(this));
+        });
+        this.input.keyboard.on('keydown-UP', () => {
+            if (this.microphoneSelected === 0) {
+                return;
+            }
+            this.microphoneSelected--;
+            mediaManager.setMicrophone(this.microphonesList[this.microphoneSelected].deviceId).then(this.setupStream.bind(this));
+        });
+
+        this.soundMeterSprite = new SoundMeterSprite(this, 50, 50);
+        this.soundMeterSprite.setVisible(false);
+        this.add.existing(this.soundMeterSprite);
+    }
+
+    /**
+     * Function called each time a camera is changed
+     */
+    private setupStream(stream: MediaStream): void {
+        const img = this.getElementByIdOrFail<HTMLDivElement>('webRtcSetupNoVideo');
+        img.style.display = 'none';
+
+        const div = this.getElementByIdOrFail<HTMLVideoElement>('myCamVideoSetup');
+        div.srcObject = stream;
+
+        this.soundMeter.connectToSource(stream, new window.AudioContext());
+
+        const bounds = div.getBoundingClientRect();
+        this.soundMeterSprite.x = this.game.renderer.width / 2 - this.soundMeterSprite.getWidth() / 2;
+        this.soundMeterSprite.y = bounds.bottom / RESOLUTION + 64;
+        this.soundMeterSprite.setVisible(true);
+
+        this.updateWebCamName();
+    }
+
+    private updateWebCamName(): void {
+        if (this.camerasList.length > 1) {
+            const div = this.getElementByIdOrFail<HTMLVideoElement>('myCamVideoSetup');
+            const bounds = div.getBoundingClientRect();
+
+            let label = this.camerasList[this.cameraSelected].label;
+            // remove text in parenthesis
+            label = label.replace(/\([^()]*\)/g, '').trim();
+            // remove accents
+            label = label.normalize("NFD").replace(/[\u0300-\u036f]/g, "");
+            this.cameraNameField.text = label;
+            this.cameraNameField.y = bounds.bottom / RESOLUTION + 30;
+
+            if (this.cameraSelected < this.camerasList.length - 1) {
+                this.arrowRight.x = bounds.right / RESOLUTION + 16;
+                this.arrowRight.y = (bounds.top + bounds.height / 2) / RESOLUTION;
+                this.arrowRight.setVisible(true);
+            } else {
+                this.arrowRight.setVisible(false);
+            }
+
+            if (this.cameraSelected > 0) {
+                this.arrowLeft.x = bounds.left / RESOLUTION - 16;
+                this.arrowLeft.y = (bounds.top + bounds.height / 2) / RESOLUTION;
+                this.arrowLeft.setVisible(true);
+            } else {
+                this.arrowLeft.setVisible(false);
+            }
+        }
+
+        if (this.microphonesList.length > 1) {
+            let label = this.microphonesList[this.microphoneSelected].label;
+            // remove text in parenthesis
+            label = label.replace(/\([^()]*\)/g, '').trim();
+            // remove accents
+            label = label.normalize("NFD").replace(/[\u0300-\u036f]/g, "");
+
+            this.microphoneNameField.text = label;
+            this.microphoneNameField.y = this.soundMeterSprite.y + 22;
+
+            if (this.microphoneSelected < this.microphonesList.length - 1) {
+                this.arrowDown.x = this.microphoneNameField.x + this.microphoneNameField.width / 2 + 16;
+                this.arrowDown.y = this.microphoneNameField.y;
+                this.arrowDown.setVisible(true);
+            } else {
+                this.arrowDown.setVisible(false);
+            }
+
+            if (this.microphoneSelected > 0) {
+                this.arrowUp.x = this.microphoneNameField.x - this.microphoneNameField.width / 2 - 16;
+                this.arrowUp.y = this.microphoneNameField.y;
+                this.arrowUp.setVisible(true);
+            } else {
+                this.arrowUp.setVisible(false);
+            }
+
+        }
+    }
+
+    update(time: number, delta: number): void {
+        this.pressReturnField.setVisible(!!(Math.floor(time / 500) % 2));
+
+        console.log(this.soundMeter.getVolume());
+        this.soundMeterSprite.setVolume(this.soundMeter.getVolume());
+    }
+
+    private async login(): Promise<StartMapInterface> {
+        this.getElementByIdOrFail<HTMLDivElement>('webRtcSetup').style.display = 'none';
+        this.soundMeter.stop();
+
+        // Do we have a start URL in the address bar? If so, let's redirect to this address
+        const instanceAndMapUrl = this.findMapUrl();
+        if (instanceAndMapUrl !== null) {
+            const [mapUrl, instance] = instanceAndMapUrl;
+            const key = gameManager.loadMap(mapUrl, this.scene, instance);
+            this.scene.start(key, {
+                startLayerName: window.location.hash ? window.location.hash.substr(1) : undefined
+            } as GameSceneInitInterface);
+            return {
+                mapUrlStart: mapUrl,
+                startInstance: instance
+            };
+        } else {
+            // If we do not have a map address in the URL, let's ask the server for a start map.
+            return gameManager.loadStartMap().then((startMap: StartMapInterface) => {
+                const key = gameManager.loadMap(window.location.protocol + "//" + startMap.mapUrlStart, this.scene, startMap.startInstance);
+                this.scene.start(key);
+                return startMap;
+            }).catch((err) => {
+                console.error(err);
+                throw err;
+            });
+        }
+    }
+
+    /**
+     * Returns the map URL and the instance from the current URL
+     */
+    private findMapUrl(): [string, string]|null {
+        const path = window.location.pathname;
+        if (!path.startsWith('/_/')) {
+            return null;
+        }
+        const instanceAndMap = path.substr(3);
+        const firstSlash = instanceAndMap.indexOf('/');
+        if (firstSlash === -1) {
+            return null;
+        }
+        const instance = instanceAndMap.substr(0, firstSlash);
+        return [window.location.protocol+'//'+instanceAndMap.substr(firstSlash+1), instance];
+    }
+
+    private async getDevices() {
+        const mediaDeviceInfos = await navigator.mediaDevices.enumerateDevices();
+        for (const mediaDeviceInfo of mediaDeviceInfos) {
+            if (mediaDeviceInfo instanceof InputDeviceInfo) {
+                if (mediaDeviceInfo.kind === 'audioinput') {
+                    this.microphonesList.push(mediaDeviceInfo);
+                } else if (mediaDeviceInfo.kind === 'videoinput') {
+                    this.camerasList.push(mediaDeviceInfo);
+                }
+            }
+        }
+        this.updateWebCamName();
+    }
+
+    private getElementByIdOrFail<T extends HTMLElement>(id: string): T {
+        const elem = document.getElementById(id);
+        if (elem === null) {
+            throw new Error("Cannot find HTML element with id '"+id+"'");
+        }
+        // FIXME: does not check the type of the returned type
+        return elem as T;
+    }
+}

front/src/Phaser/Login/SelectCharacterScene.ts

@@ -6,6 +6,7 @@ import Rectangle = Phaser.GameObjects.Rectangle;
 import {PLAYER_RESOURCES, PlayerResourceDescriptionInterface} from "../Entity/Character";
 import {GameSceneInitInterface} from "../Game/GameScene";
 import {StartMapInterface} from "../../Connection";
+import {EnableCameraSceneName} from "./EnableCameraScene";
 
 //todo: put this constants in a dedicated file
 export const SelectCharacterSceneName = "SelectCharacterScene";
@@ -116,11 +117,13 @@ export class SelectCharacterScene extends Phaser.Scene {
         this.pressReturnField.setVisible(!!(Math.floor(time / 500) % 2));
     }
 
-    private async login(name: string): Promise<StartMapInterface> {
+    private login(name: string): void {
         gameManager.storePlayerDetails(name, this.selectedPlayer.texture.key);
 
+        this.scene.start(EnableCameraSceneName);
+
         // Do we have a start URL in the address bar? If so, let's redirect to this address
-        const instanceAndMapUrl = this.findMapUrl();
+        /*const instanceAndMapUrl = this.findMapUrl();
         if (instanceAndMapUrl !== null) {
             const [mapUrl, instance] = instanceAndMapUrl;
             const key = gameManager.loadMap(mapUrl, this.scene, instance);
@@ -141,7 +144,7 @@ export class SelectCharacterScene extends Phaser.Scene {
                 console.error(err);
                 throw err;
             });
-        }
+        }*/
     }
 
     /**

front/src/WebRtc/MediaManager.ts

@@ -6,7 +6,9 @@ const videoConstraint: boolean|MediaTrackConstraints = {
 
 type UpdatedLocalStreamCallback = (media: MediaStream) => void;
 
-class MediaManager {
+// TODO: Split MediaManager in 2 classes: MediaManagerUI (in charge of HTML) and MediaManager (singleton in charge of the camera only)
+// TODO: verify that microphone event listeners are not triggered plenty of time NOW (since MediaManager is created many times!!!!)
+export class MediaManager {
     localStream: MediaStream|null = null;
     private remoteVideo: Map<string, HTMLVideoElement> = new Map<string, HTMLVideoElement>();
     myCamVideo: HTMLVideoElement;
@@ -154,6 +156,26 @@ class MediaManager {
         return promise;
     }
 
+    setCamera(id: string): Promise<MediaStream> {
+        let video = this.constraintsMedia.video;
+        if (typeof(video) === 'boolean' || video === undefined) {
+            video = {}
+        }
+        video.deviceId = id;
+
+        return this.getCamera();
+    }
+
+    setMicrophone(id: string): Promise<MediaStream> {
+        let audio = this.constraintsMedia.audio;
+        if (typeof(audio) === 'boolean' || audio === undefined) {
+            audio = {}
+        }
+        audio.deviceId = id;
+
+        return this.getCamera();
+    }
+
     /**
      *
      * @param userId

front/src/index.ts

@@ -6,13 +6,14 @@ import {LoginScene} from "./Phaser/Login/LoginScene";
 import {ReconnectingScene} from "./Phaser/Reconnecting/ReconnectingScene";
 import {gameManager} from "./Phaser/Game/GameManager";
 import {SelectCharacterScene} from "./Phaser/Login/SelectCharacterScene";
+import {EnableCameraScene} from "./Phaser/Login/EnableCameraScene";
 
 const config: GameConfig = {
     title: "Office game",
     width: window.innerWidth / RESOLUTION,
     height: window.innerHeight / RESOLUTION,
     parent: "game",
-    scene: [LoginScene, SelectCharacterScene, ReconnectingScene],
+    scene: [LoginScene, SelectCharacterScene, EnableCameraScene, ReconnectingScene],
     zoom: RESOLUTION,
     physics: {
         default: "arcade",