1
0

Working on audio plumbing

Signed-off-by: James Ketrenos <james_eikona@ketrenos.com>
This commit is contained in:
James Ketrenos 2022-03-07 19:31:08 -08:00
parent 39c74a7e8d
commit 44c2a22ff1
8 changed files with 528 additions and 44 deletions

View File

@ -22,10 +22,11 @@
"react-moment": "^1.1.1",
"react-router-dom": "^6.2.1",
"react-scripts": "5.0.0",
"socket.io-client": "^4.4.1",
"web-vitals": "^2.1.2"
},
"scripts": {
"start": "react-scripts start",
"start": "HTTPS=true react-scripts start",
"build": "export $(cat ../.env | xargs) && react-scripts build",
"test": "export $(cat ../.env | xargs) && react-scripts test",
"eject": "export $(cat ../.env | xargs) && react-scripts eject"

View File

@ -0,0 +1,21 @@
.MediaAgent {
display: none;
position: absolute;
display: flex;
top: 0;
left: 0;
z-index: 50000;
}
.MediaControl {
display: flex;
position: relative;
flex-direction: row;
flex-grow: 1;
justify-content: flex-end;
align-items: center;
}
.MediaControl > div {
display: flex;
}

351
client/src/MediaControl.js Normal file
View File

@ -0,0 +1,351 @@
import React, { useState, useEffect } from "react";
import "./MediaControl.css";
import VolumeOff from '@mui/icons-material/VolumeOff';
import VolumeUp from '@mui/icons-material/VolumeUp';
import MicOff from '@mui/icons-material/MicOff';
import Mic from '@mui/icons-material/Mic';
const USE_AUDIO = true;
const DEFAULT_CHANNEL = 'some-global-channel-name';
const MUTE_AUDIO_BY_DEFAULT = false;
let local_media_stream = undefined;
const ICE_SERVERS = [
{urls:"stun:stun.l.google.com:19302"}
];
const MediaAgent = ({ table }) => {
const [peers, setPeers] = useState({});
if (!table.ws) {
return <></>;
}
const ws = table.ws;
const addPeer = (config) => {
console.log('Signaling server said to add peer:', config);
const peer_id = config.peer_id;
if (peer_id in peers) {
/* This could happen if the user joins multiple channels where the other peer is also in. */
console.log("Already connected to peer ", peer_id);
return;
}
const peer_connection = new RTCPeerConnection(
{ iceServers: ICE_SERVERS },
/* this will no longer be needed by chrome
* eventually (supposedly), but is necessary
* for now to get firefox to talk to chrome */
{ optional: [{DtlsSrtpKeyAgreement: true}]}
);
peers[peer_id] = peer_connection;
setPeers(Object.assign({}, peers));
peer_connection.onicecandidate = (event) => {
if (event.candidate) {
ws.send(JSON.stringify({
type: 'relayICECandidate',
config: {
peer_id: peer_id,
ice_candidate: {
sdpMLineIndex: event.candidate.sdpMLineIndex,
candidate: event.candidate.candidate
}
}
}));
}
};
peer_connection.ontrack = (event) => {
console.log("ontrack", event);
peer_connection.attributes = {
autoPlay: 'autoplay',
muted: true,
controls: true
};
if (window.URL) {
peer_connection.extra = {
srcObject: event.streams[0]
};
} else {
peer_connection.extra = {
src: event.streams[0]
};
}
};
/* Add our local stream */
peer_connection.addStream(local_media_stream);
/* Only one side of the peer connection should create the
* offer, the signaling server picks one to be the offerer.
* The other user will get a 'sessionDescription' event and will
* create an offer, then send back an answer 'sessionDescription' to us
*/
if (config.should_create_offer) {
console.log("Creating RTC offer to ", peer_id);
peer_connection.createOffer((local_description) => {
console.log("Local offer description is: ", local_description);
peer_connection.setLocalDescription(local_description, () => {
ws.send(JSON.stringify({
type: 'relaySessionDescription',
config: {
'peer_id': peer_id,
'session_description': local_description
}
}));
console.log("Offer setLocalDescription succeeded");
}, () => {
console.error("Offer setLocalDescription failed!");
})
}, (error) => {
console.log("Error sending offer: ", error);
});
}
}
const sessionDescription = ({ peer_id, session_description }) => {
/**
* Peers exchange session descriptions which contains information
* about their audio / video settings and that sort of stuff. First
* the 'offerer' sends a description to the 'answerer' (with type
* "offer"), then the answerer sends one back (with type "answer").
*/
console.log('Remote description received: ', peer_id, session_description);
const peer = peers[peer_id];
console.log(session_description);
const desc = new RTCSessionDescription(session_description);
const stuff = peer.setRemoteDescription(desc, () => {
console.log("setRemoteDescription succeeded");
if (session_description.type == "offer") {
console.log("Creating answer");
peer.createAnswer((local_description) => {
console.log("Answer description is: ", local_description);
peer.setLocalDescription(local_description, () => {
ws.send(JSON.stringify({
type: 'relaySessionDescription',
config: {
peer_id,
session_description: local_description
}
}));
console.log("Answer setLocalDescription succeeded");
}, () => {
console.error("Answer setLocalDescription failed!");
});
}, (error) => {
console.log("Error creating answer: ", error);
console.log(peer);
});
}
}, (error) => {
console.log("setRemoteDescription error: ", error);
});
console.log("Description Object: ", desc);
};
const removePeer = ({peer_id}) => {
console.log('Signaling server said to remove peer:', peer_id);
if (peer_id in peers) {
peers[peer_id].close();
}
delete peers[peer_id];
setPeers(Object.assign({}, peers));
};
const iceCandidate = (config) => {
/**
* The offerer will send a number of ICE Candidate blobs to the answerer so they
* can begin trying to find the best path to one another on the net.
*/
const peer = peers[config.peer_id],
ice_candidate = config.ice_candidate;
if (!peer) {
console.error(`No peer for ${config.peer_id}`);
return;
}
peer.addIceCandidate(new RTCIceCandidate(ice_candidate))
.then(() => {
})
.catch((error) => {
console.error(peer, ice_candidate);
console.error(error);
});
};
ws.addEventListener('message', (event) => {
let data;
try {
data = JSON.parse(event.data);
} catch (error) {
console.error(error);
return;
}
switch (data.type) {
case 'addPeer': addPeer(data.data); break;
case 'removePeer': removePeer(data.data); break;
case 'iceCandidate': iceCandidate(data.data); break;
case 'sessionDescription': sessionDescription(data.data); break;
}
});
ws.addEventListener('error', (event) => {
console.error(`${table.game.name} WebSocket error`);
});
ws.addEventListener('close', (event) => {
console.log(`${table.game.name} Disconnected from signaling server`);
/* Tear down all of our peer connections and remove all the
* media divs when we disconnect */
for (let peer_id in peers) {
peers[peer_id].close();
}
for (let id in peers) {
delete peers[id];
}
setPeers(Object.assign({}, peers));
});
ws.addEventListener('open', (event) => {
setup_local_media(() => {
/* once the user has given us access to their
* microphone/camcorder, join the channel and start peering up */
join_chat_channel(ws, table.game.id, {'whatever-you-want-here': 'stuff'});
});
});
if (!table.game) {
return <></>;
}
const setup_local_media = (callback, errorback) => {
if (local_media_stream !== undefined) { /* ie, if we've already been initialized */
if (callback) callback();
return;
}
/* Ask user for permission to use the computers microphone and/or camera,
* attach it to an <audio> or <video> tag if they give us access. */
console.log("Requesting access to local audio / video inputs");
navigator.getUserMedia = (navigator.getUserMedia ||
navigator.webkitGetUserMedia ||
navigator.mozGetUserMedia ||
navigator.msGetUserMedia);
navigator.mediaDevices.getUserMedia({audio: true, video: false})//, "video": true})
.then((stream) => { /* user accepted access to a/v */
console.log("Access granted to audio/video");
local_media_stream = stream;
var local_media = document.querySelector('audio.local');
if (!local_media) {
console.error(`No local media!`);
}
if (window.URL) {
local_media.srcobject = stream;
} else {
local_media.src = stream;
}
if (callback) callback();
})
.catch((error) => { /* user denied access to a/v */
console.error(error);
console.log("Access denied for audio/video");
window.alert("You chose not to provide access to the microphone!" +
"Ketran will have no audio :(");
if (errorback) errorback();
});
};
const join_chat_channel = (ws, channel, userdata) => {
ws.send(JSON.stringify({
type: 'join',
config: { channel, userdata }
}));
}
const part_chat_channel = (channel) => {
ws.send(JSON.stringify({ type: 'part', config: channel }));
}
const audioPeers = [];
for (let id in peers) {
const peer = peers[id];
if (peer.attributes) {
audioPeers.push(<audio
ref={el => {
if (!el) {
return;
}
if (peer.extra.srcObject) {
el.srcObject = peer.extra.srcObject;
} else {
el.src = peer.extra.src;
}
}}
key={`Peer-${id}`} className="Remote" {...peer.attributes}/>);
}
}
return <div className="MediaAgent">
<audio className="local" autoPlay="autoplay" muted={true} controls/>
{ audioPeers }
</div>;
}
const MediaControl = ({table, color }) => {
const [mute, setMute] = useState(false);
const [mic, setMic] = useState(true);
const [noAudio, setNoAudio] = useState(false);
const [rtc, setRtc] = useState(undefined);
if (!table || !table.game) {
return <></>;
}
/*
const player = table.game.player;
if (!player) {
return <></>;
}
*/
if (table.game.players[color].status !== 'Active') {
return <div className="MediaControl"/>;
}
const isSelf = table.game.color === color;
if (!rtc) {
// setRtc(RTC({ id: table.game.id }));
}
const toggleMic = (event) => {
setMic(!mic);
event.stopPropagation();
}
const toggleMute = (event) => {
setMute(!mute);
event.stopPropagation();
}
return (<div className="MediaControl">
{ isSelf && <div onClick={toggleMic}>
{ mic && <MicOff color={noAudio ? 'disabled' : 'primary'}/> }
{ !mic && <Mic color={noAudio ? 'disabled' : 'primary'}/> }
</div> }
<div onClick={toggleMute}>
{ mute && <VolumeOff color={noAudio ? 'disabled' : 'primary'}/> }
{ !mute && <VolumeUp color={noAudio ? 'disabled' : 'primary'}/> }
</div>
</div>);
};
export { MediaControl, MediaAgent };

View File

@ -260,14 +260,6 @@
padding: 0;
}
.PlayerSelector .PlayerEntry {
flex: 1 1 0px;
align-items: center;
display: inline-flex;
flex-direction: row;
min-width: 10em;
}
.PlayerSelector .MuiTypography-body1 {
font-size: 0.8rem;
/* white-space: nowrap;*/
@ -282,6 +274,15 @@
border: 1px solid rgba(0,0,0,0);
border-radius: 0.5em;
min-width: 10em;
justify-content: space-between;
}
.PlayerSelector .PlayerEntry {
flex: 1 1 0px;
align-items: center;
display: inline-flex;
flex-direction: row;
min-width: 10em;
}
.Players .PlayerEntry[data-selectable=true]:hover {
@ -293,6 +294,7 @@
background-color: rgba(255, 255, 0, 0.5);
}
.Players .PlayerToggle {
min-width: 5em;
display: inline-flex;

View File

@ -7,7 +7,6 @@ import TextField from '@material-ui/core/TextField';
import List from '@material-ui/core/List';
import Board from './Board.js';
import Trade from './Trade.js';
import { assetsPath, base, getPlayerName, gamesPath } from './Common.js';
import PlayerColor from './PlayerColor.js';
import Dice from './Dice.js';
import Resource from './Resource.js';
@ -20,6 +19,8 @@ import 'moment-timezone';
import Activities from './Activities.js';
import Placard from './Placard.js';
import PlayersStatus from './PlayersStatus.js';
import { MediaAgent, MediaControl } from './MediaControl.js';
import { base, assetsPath, getPlayerName, gamesPath } from './Common.js';
/* Start of withRouter polyfill */
// https://reactrouter.com/docs/en/v6/faq#what-happened-to-withrouter-i-need-it
@ -300,6 +301,7 @@ const Players = ({ table }) => {
onClick={() => { inLobby && selectable && toggleSelected(color) }}
key={`player-${color}`}>
<PlayerColor color={color}/>{name}
<MediaControl color={color} table={table}/>
</div>
));
}
@ -672,6 +674,10 @@ class Table extends React.Component {
resetKeepAlive(isDead) {
if (isDead) {
console.log(`Short circuiting keep-alive`);
if (this.ws) {
this.ws.close();
delete this.ws;
}
} else {
console.log(`${this.game.name} Resetting keep-alive. Last ping: ${(Date.now() - this.lastPing) / 1000}`);
}
@ -688,6 +694,7 @@ class Table extends React.Component {
this.setState({ noNetwork: true });
if (this.ws) {
this.ws.close();
delete this.ws;
}
this.connectWebSocket();
}, isDead ? 3000 : 10000);
@ -705,14 +712,16 @@ class Table extends React.Component {
this.resetKeepAlive(true);
return;
}
if (this.ws) {
return;
}
let loc = window.location, new_uri;
if (loc.protocol === "https:") {
new_uri = "wss";
} else {
new_uri = "ws";
}
new_uri = `${new_uri}://${loc.host}${base}/api/v1/games/ws/${this.game.id}`;
new_uri = `${new_uri}://${loc.host}${base}/api/v1/games/ws/${this.game.id}/`;
console.log(`Attempting WebSocket connection to ${new_uri}`);
this.ws = new WebSocket(new_uri);
@ -743,7 +752,6 @@ class Table extends React.Component {
this.ws.send(JSON.stringify({ type: 'pong', timestamp: data.ping }));
break;
default:
console.log(`Unknown event type: ${data.type}`);
break;
}
});
@ -765,7 +773,6 @@ class Table extends React.Component {
this.ws.send(JSON.stringify({ type: 'game-update' }));
this.resetKeepAlive();
});
}
componentDidMount() {
@ -925,7 +932,8 @@ class Table extends React.Component {
return <></>;
}
return (
return (<>
<MediaAgent table={this}/>
<div className="Table">
{ this.state.loading > 0 && <CircularProgress className='Loading'/> }
@ -1018,7 +1026,7 @@ class Table extends React.Component {
{ this.state.error && <Paper onClick={() => this.setState({ error: undefined })} className="Error"><div>{this.state.error}</div></Paper> }
</div>
);
</>);
}
}
export default withRouter(props => <Table {...props}/>);

View File

@ -2,7 +2,7 @@ const { createProxyMiddleware } = require('http-proxy-middleware');
module.exports = function(app) {
const base = process.env.PUBLIC_URL;
console.log('http-proxy-middleware');
console.log(`http-proxy-middleware ${base}`);
app.use(createProxyMiddleware(
`${base}/api/v1/games/ws`, {
ws: true,

View File

@ -12,8 +12,20 @@ const express = require("express"),
SQLiteStore = require('connect-sqlite3')(session),
basePath = require("./basepath"),
app = express(),
server = require("http").createServer(app),
ws = require('express-ws')(app, server);
fs = require('fs');
let server;
if (0) {
const key = fs.readFileSync("ssl/server-key.pem", "utf8"),
cert = fs.readFileSync("ssl/server-cert.pem", "utf8"),
credentials = { key, cert };
server = require("https").createServer(credentials, app);
} else {
server = require("http").createServer(app);
}
const ws = require('express-ws')(app, server);
require("./console-line.js"); /* Monkey-patch console.log with line numbers */

View File

@ -2719,9 +2719,43 @@ const resetDisconnectCheck = (game, req) => {
//req.disconnectCheck = setTimeout(() => { wsInactive(game, req) }, 20000);
}
const channels = {},
sockets = {};
const part = (ws, channel) => {
console.log(`[${ws.id}] part `);
if (!(channel in ws.channels)) {
console.log(`[${ws.id}] ERROR: not in `, channel);
return;
}
delete ws.channels[channel];
delete channels[channel][id];
for (id in channels[channel]) {
channels[channel][id].send(JSON.stringify({
type: 'removePeer',
data: {'peer_id': id}
}));
ws.send(JSON.stringify({
type: 'removePeer',
data: {'peer_id': id}
}));
}
};
router.ws("/ws/:id", async (ws, req) => {
const { id } = req.params;
ws.id = req.session.player_id;
ws.channels = {},
sockets[ws.id] = ws;
console.log(`Connection for ${ws.id}`);
/* Setup WebSocket event handlers prior to performing any async calls or
* we may miss the first messages from clients */
ws.on('error', async (event) => {
@ -2754,32 +2788,87 @@ router.ws("/ws/:id", async (ws, req) => {
session.ws = undefined;
}
}
/* Cleanup any voice channels */
for (let channel in ws.channels) {
part(channel);
}
console.log(`[${ws.id}] disconnected`);
delete sockets[ws.id];
});
ws.on('message', async (message) => {
/* Ensure the session is loaded prior to the first 'message'
* being processed */
const data = JSON.parse(message);
switch (data.type) {
case 'join': {
const { channel, userdata } = data.config;
console.log(`[${ws.id}] join `, channel);
if (channel in ws.channels) {
console.log(`[${ws.id}] ERROR: already joined `, channel);
return;
}
if (!(channel in channels)) {
channels[channel] = {};
}
for (let id in channels[channel]) {
channels[channel][id].send(JSON.stringify({
type: 'addPeer',
data: { 'peer_id': id, 'should_create_offer': false }
}));
ws.send(JSON.stringify({
type: 'addPeer',
data: {'peer_id': id, 'should_create_offer': true}
}));
}
channels[channel][ws.id] = ws;
ws.channels[channel] = channel;
} break;
case 'part':
part(ws, data.part);
break;
case 'relayICECandidate': {
const { peer_id, ice_candidate } = data.config;
console.log(`[${ws.id}] relaying ICE candidate to [${peer_id}] `, ice_candidate);
if (peer_id in sockets) {
sockets[peer_id].send(JSON.stringify({
type: 'iceCandidate',
data: {'peer_id': ws.id, 'ice_candidate': ice_candidate }
}));
}
} break;
case 'relaySessionDescription': {
const { peer_id, session_description } = data.config;
console.log(`[${ws.id}] relaying session description to [${peer_id}] `, session_description);
if (peer_id in sockets) {
sockets[peer_id].send(JSON.stringify({
type: 'sessionDescription',
data: {'peer_id': ws.id, 'session_description': session_description }
}));
}
} break;
case 'pong': {
const game = await loadGame(id);
if (!game) {
console.error(`Unable to load/create new game for WS request.`);
return;
}
const session = getSession(game, req.session);
try {
const data = JSON.parse(message);
switch (data.type) {
case 'pong':
// console.log(`Latency for ${session.name ? session.name : 'Unammed'} is ${Date.now() - data.timestamp}`);
resetDisconnectCheck(game, req);
break;
case 'game-update':
} break;
case 'game-update': {
const game = await loadGame(id);
if (!game) {
console.error(`Unable to load/create new game for WS request.`);
return;
}
console.log(`Player ${session.name ? session.name : 'Unnamed'} requested a game update.`);
resetDisconnectCheck(game, req);
sendGame(req, undefined, game, undefined, ws);
break;
}
} catch (error) {
console.error(error);
} break;
}
});