Merge pull request #488 from thecodingmachine/apiscaling

Add scaling capability to the API servers with this one weird trick
This commit is contained in:
David Négrier 2020-12-16 10:54:12 +01:00 committed by GitHub
commit 95a42197a6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 77 additions and 12 deletions

View File

@ -3,11 +3,13 @@ import App from "./src/App";
import grpc from "grpc"; import grpc from "grpc";
import {roomManager} from "./src/RoomManager"; import {roomManager} from "./src/RoomManager";
import {IRoomManagerServer, RoomManagerService} from "./src/Messages/generated/messages_grpc_pb"; import {IRoomManagerServer, RoomManagerService} from "./src/Messages/generated/messages_grpc_pb";
import {HTTP_PORT, GRPC_PORT} from "./src/Enum/EnvironmentVariable";
App.listen(8080, () => console.log(`WorkAdventure starting on port 8080!`)) App.listen(HTTP_PORT, () => console.log(`WorkAdventure HTTP API starting on port %d!`, HTTP_PORT))
const server = new grpc.Server(); const server = new grpc.Server();
server.addService<IRoomManagerServer>(RoomManagerService, roomManager); server.addService<IRoomManagerServer>(RoomManagerService, roomManager);
server.bind('0.0.0.0:50051', grpc.ServerCredentials.createInsecure()); server.bind('0.0.0.0:'+GRPC_PORT, grpc.ServerCredentials.createInsecure());
server.start(); server.start();
console.log('WorkAdventure HTTP/2 API starting on port %d!', GRPC_PORT);

View File

@ -10,7 +10,9 @@ const CPU_OVERHEAT_THRESHOLD = Number(process.env.CPU_OVERHEAT_THRESHOLD) || 80;
const JITSI_URL : string|undefined = (process.env.JITSI_URL === '') ? undefined : process.env.JITSI_URL; const JITSI_URL : string|undefined = (process.env.JITSI_URL === '') ? undefined : process.env.JITSI_URL;
const JITSI_ISS = process.env.JITSI_ISS || ''; const JITSI_ISS = process.env.JITSI_ISS || '';
const SECRET_JITSI_KEY = process.env.SECRET_JITSI_KEY || ''; const SECRET_JITSI_KEY = process.env.SECRET_JITSI_KEY || '';
export const SOCKET_IDLE_TIMER = parseInt(process.env.SOCKET_IDLE_TIMER as string) || 30; // maximum time (in second) without activity before a socket is closed const HTTP_PORT = parseInt(process.env.HTTP_PORT || '8080') || 8080;
const GRPC_PORT = parseInt(process.env.GRPC_PORT || '50051') || 50051;
export const SOCKET_IDLE_TIMER = parseInt(process.env.SOCKET_IDLE_TIMER as string) || 30; // maximum time (in second) without activity before a socket is closed
export { export {
SECRET_KEY, SECRET_KEY,
@ -18,6 +20,8 @@ export {
MINIMUM_DISTANCE, MINIMUM_DISTANCE,
ADMIN_API_URL, ADMIN_API_URL,
ADMIN_API_TOKEN, ADMIN_API_TOKEN,
HTTP_PORT,
GRPC_PORT,
MAX_USERS_PER_ROOM, MAX_USERS_PER_ROOM,
GROUP_RADIUS, GROUP_RADIUS,
ALLOW_ARTILLERY, ALLOW_ARTILLERY,

View File

@ -7,10 +7,10 @@
"$schema": "https://raw.githubusercontent.com/thecodingmachine/deeployer/master/deeployer.schema.json", "$schema": "https://raw.githubusercontent.com/thecodingmachine/deeployer/master/deeployer.schema.json",
"version": "1.0", "version": "1.0",
"containers": { "containers": {
"back": { "back1": {
"image": "thecodingmachine/workadventure-back:"+tag, "image": "thecodingmachine/workadventure-back:"+tag,
"host": { "host": {
"url": "api."+url, "url": "api1."+url,
"https": "enable", "https": "enable",
"containerPort": 8080 "containerPort": 8080
}, },
@ -25,6 +25,24 @@
"ADMIN_API_URL": adminUrl, "ADMIN_API_URL": adminUrl,
} else {} } else {}
}, },
"back2": {
"image": "thecodingmachine/workadventure-back:"+tag,
"host": {
"url": "api2."+url,
"https": "enable",
"containerPort": 8080
},
"ports": [8080, 50051],
"env": {
"SECRET_KEY": "tempSecretKeyNeedsToChange",
"ADMIN_API_TOKEN": env.ADMIN_API_TOKEN,
"JITSI_ISS": env.JITSI_ISS,
"JITSI_URL": env.JITSI_URL,
"SECRET_JITSI_KEY": env.SECRET_JITSI_KEY,
} + if adminUrl != null then {
"ADMIN_API_URL": adminUrl,
} else {}
},
"pusher": { "pusher": {
"replicas": 2, "replicas": 2,
"image": "thecodingmachine/workadventure-pusher:"+tag, "image": "thecodingmachine/workadventure-pusher:"+tag,
@ -38,7 +56,7 @@
"ADMIN_API_TOKEN": env.ADMIN_API_TOKEN, "ADMIN_API_TOKEN": env.ADMIN_API_TOKEN,
"JITSI_ISS": env.JITSI_ISS, "JITSI_ISS": env.JITSI_ISS,
"JITSI_URL": env.JITSI_URL, "JITSI_URL": env.JITSI_URL,
"API_URL": "back:50051", "API_URL": "back1:50051,back2:50051",
"SECRET_JITSI_KEY": env.SECRET_JITSI_KEY, "SECRET_JITSI_KEY": env.SECRET_JITSI_KEY,
} + if adminUrl != null then { } + if adminUrl != null then {
"ADMIN_API_URL": adminUrl, "ADMIN_API_URL": adminUrl,
@ -100,7 +118,35 @@
}, },
k8sextension(k8sConf):: k8sextension(k8sConf)::
k8sConf + { k8sConf + {
back+: { back1+: {
deployment+: {
spec+: {
template+: {
metadata+: {
annotations+: {
"prometheus.io/port": "8080",
"prometheus.io/scrape": "true"
}
}
}
}
}
},
back2+: {
deployment+: {
spec+: {
template+: {
metadata+: {
annotations+: {
"prometheus.io/port": "8080",
"prometheus.io/scrape": "true"
}
}
}
}
}
},
pusher+: {
deployment+: { deployment+: {
spec+: { spec+: {
template+: { template+: {

View File

@ -3,17 +3,30 @@
*/ */
import {RoomManagerClient} from "../Messages/generated/messages_grpc_pb"; import {RoomManagerClient} from "../Messages/generated/messages_grpc_pb";
import grpc from 'grpc'; import grpc from 'grpc';
import crypto from 'crypto';
import {API_URL} from "../Enum/EnvironmentVariable"; import {API_URL} from "../Enum/EnvironmentVariable";
import Debug from "debug";
const debug = Debug('apiClientRespository');
class ApiClientRepository { class ApiClientRepository {
private roomManagerClient: RoomManagerClient|null = null; private roomManagerClients: RoomManagerClient[] = [];
public constructor(private apiUrls: string[]) {
}
public async getClient(roomId: string): Promise<RoomManagerClient> { public async getClient(roomId: string): Promise<RoomManagerClient> {
if (this.roomManagerClient === null) { const array = new Uint32Array(crypto.createHash('md5').update(roomId).digest());
this.roomManagerClient = new RoomManagerClient(API_URL, grpc.credentials.createInsecure()); const index = array[0] % this.apiUrls.length;
let client = this.roomManagerClients[index];
if (client === undefined) {
this.roomManagerClients[index] = client = new RoomManagerClient(this.apiUrls[index], grpc.credentials.createInsecure());
debug('Mapping room %s to API server %s', roomId, this.apiUrls[index])
} }
return Promise.resolve(this.roomManagerClient);
return Promise.resolve(client);
} }
public async getAllClients(): Promise<RoomManagerClient[]> { public async getAllClients(): Promise<RoomManagerClient[]> {
@ -21,6 +34,6 @@ class ApiClientRepository {
} }
} }
const apiClientRepository = new ApiClientRepository(); const apiClientRepository = new ApiClientRepository(API_URL.split(','));
export { apiClientRepository }; export { apiClientRepository };