Working on audio plumbing

Signed-off-by: James Ketrenos <james_eikona@ketrenos.com>

client/package.json

@@ -22,10 +22,11 @@
     "react-moment": "^1.1.1",
     "react-router-dom": "^6.2.1",
     "react-scripts": "5.0.0",
+    "socket.io-client": "^4.4.1",
     "web-vitals": "^2.1.2"
   },
   "scripts": {
-    "start": "react-scripts start",
+    "start": "HTTPS=true react-scripts start",
     "build": "export $(cat ../.env | xargs) && react-scripts build",
     "test": "export $(cat ../.env | xargs) && react-scripts test",
     "eject": "export $(cat ../.env | xargs) && react-scripts eject"

client/src/MediaControl.css

@@ -0,0 +1,21 @@
+.MediaAgent {
+  display: none;
+  position: absolute;
+  display: flex;
+  top: 0;
+  left: 0;
+  z-index: 50000;
+}
+
+.MediaControl {
+  display: flex;
+  position: relative;
+  flex-direction: row;
+  flex-grow: 1;
+  justify-content: flex-end;
+  align-items: center;
+}
+
+.MediaControl > div {
+  display: flex;
+}

client/src/MediaControl.js

@@ -0,0 +1,351 @@
+import React, { useState, useEffect } from "react";
+import "./MediaControl.css";
+
+import VolumeOff from '@mui/icons-material/VolumeOff';
+import VolumeUp from '@mui/icons-material/VolumeUp';
+import MicOff from '@mui/icons-material/MicOff';
+import Mic from '@mui/icons-material/Mic';
+
+const USE_AUDIO = true;
+const DEFAULT_CHANNEL = 'some-global-channel-name';
+const MUTE_AUDIO_BY_DEFAULT = false;
+let local_media_stream = undefined;
+
+const ICE_SERVERS = [
+  {urls:"stun:stun.l.google.com:19302"}
+];
+
+const MediaAgent = ({ table }) => {
+  const [peers, setPeers] = useState({});
+
+  if (!table.ws) {
+    return <></>;
+  }
+
+  const ws = table.ws;
+
+  const addPeer = (config) => {
+    console.log('Signaling server said to add peer:', config);
+    const peer_id = config.peer_id;
+    if (peer_id in peers) {
+      /* This could happen if the user joins multiple channels where the other peer is also in. */
+      console.log("Already connected to peer ", peer_id);
+      return;
+    }
+    const peer_connection = new RTCPeerConnection(
+      { iceServers: ICE_SERVERS },
+       /* this will no longer be needed by chrome
+        * eventually (supposedly), but is necessary 
+        * for now to get firefox to talk to chrome */
+       { optional: [{DtlsSrtpKeyAgreement: true}]}
+    );
+    peers[peer_id] = peer_connection;
+    setPeers(Object.assign({}, peers));
+    
+    peer_connection.onicecandidate = (event) => {
+      if (event.candidate) {
+        ws.send(JSON.stringify({
+          type: 'relayICECandidate', 
+          config: {
+            peer_id: peer_id, 
+            ice_candidate: {
+              sdpMLineIndex: event.candidate.sdpMLineIndex,
+              candidate: event.candidate.candidate
+            }
+          }
+        }));
+      }
+    };
+
+    peer_connection.ontrack = (event) => {
+      console.log("ontrack", event);
+      peer_connection.attributes = {
+        autoPlay: 'autoplay',
+        muted: true,
+        controls: true
+      };
+      if (window.URL) {
+        peer_connection.extra = {
+          srcObject: event.streams[0]
+        };
+      } else {
+        peer_connection.extra = {
+          src: event.streams[0]
+        };
+      }
+    };
+
+    /* Add our local stream */
+    peer_connection.addStream(local_media_stream);
+
+    /* Only one side of the peer connection should create the
+     * offer, the signaling server picks one to be the offerer. 
+     * The other user will get a 'sessionDescription' event and will
+     * create an offer, then send back an answer 'sessionDescription' to us
+     */
+    if (config.should_create_offer) {
+      console.log("Creating RTC offer to ", peer_id);
+      peer_connection.createOffer((local_description) => { 
+        console.log("Local offer description is: ", local_description);
+        peer_connection.setLocalDescription(local_description, () => { 
+          ws.send(JSON.stringify({
+            type: 'relaySessionDescription', 
+            config: { 
+              'peer_id': peer_id,
+              'session_description': local_description
+            }
+          }));
+          console.log("Offer setLocalDescription succeeded"); 
+        }, () => { 
+          console.error("Offer setLocalDescription failed!"); 
+        })
+      }, (error) => {
+        console.log("Error sending offer: ", error);
+      });
+    }
+  }
+  
+  const sessionDescription = ({ peer_id, session_description }) => {
+  /** 
+     * Peers exchange session descriptions which contains information
+     * about their audio / video settings and that sort of stuff. First
+     * the 'offerer' sends a description to the 'answerer' (with type
+     * "offer"), then the answerer sends one back (with type "answer").  
+     */
+    console.log('Remote description received: ', peer_id, session_description);
+    const peer = peers[peer_id];
+    console.log(session_description);
+
+    const desc = new RTCSessionDescription(session_description);
+    const stuff = peer.setRemoteDescription(desc, () => {
+      console.log("setRemoteDescription succeeded");
+      if (session_description.type == "offer") {
+        console.log("Creating answer");
+        peer.createAnswer((local_description) => {
+          console.log("Answer description is: ", local_description);
+          peer.setLocalDescription(local_description, () => { 
+            ws.send(JSON.stringify({
+              type: 'relaySessionDescription',
+              config: {
+                peer_id, 
+                session_description: local_description
+              }
+            }));
+            console.log("Answer setLocalDescription succeeded");
+          }, () => {
+            console.error("Answer setLocalDescription failed!");
+          });
+        }, (error) => {
+          console.log("Error creating answer: ", error);
+          console.log(peer);
+        });
+      }
+    }, (error) => {
+      console.log("setRemoteDescription error: ", error);
+    });
+    
+    console.log("Description Object: ", desc);
+  };
+
+  const removePeer = ({peer_id}) => {
+    console.log('Signaling server said to remove peer:', peer_id);
+    if (peer_id in peers) {
+      peers[peer_id].close();
+    }
+  
+    delete peers[peer_id];
+    setPeers(Object.assign({}, peers));
+  };
+
+  const iceCandidate = (config) => {
+    /**
+    * The offerer will send a number of ICE Candidate blobs to the answerer so they 
+    * can begin trying to find the best path to one another on the net.
+    */
+    const peer = peers[config.peer_id], 
+      ice_candidate = config.ice_candidate;
+    if (!peer) {
+      console.error(`No peer for ${config.peer_id}`);
+      return;
+    }
+    peer.addIceCandidate(new RTCIceCandidate(ice_candidate))
+    .then(() => {
+    })
+    .catch((error) => {
+      console.error(peer, ice_candidate);
+      console.error(error);
+    });
+  };
+ 
+  ws.addEventListener('message', (event) => {  
+    let data;
+    try {
+      data = JSON.parse(event.data);
+    } catch (error) {
+      console.error(error);
+      return;
+    }
+    switch (data.type) {
+    case 'addPeer': addPeer(data.data); break;
+    case 'removePeer': removePeer(data.data); break;
+    case 'iceCandidate': iceCandidate(data.data); break;
+    case 'sessionDescription': sessionDescription(data.data); break;
+    }
+  });
+
+  ws.addEventListener('error', (event) => {
+    console.error(`${table.game.name} WebSocket error`);
+  });
+  
+  ws.addEventListener('close', (event) => {
+    console.log(`${table.game.name} Disconnected from signaling server`);
+    /* Tear down all of our peer connections and remove all the
+      * media divs when we disconnect */
+    for (let peer_id in peers) {
+      peers[peer_id].close();
+    }
+
+    for (let id in peers) {
+      delete peers[id];
+    }
+    setPeers(Object.assign({}, peers));
+  });
+  
+  ws.addEventListener('open', (event) => {
+    setup_local_media(() => {
+      /* once the user has given us access to their
+       * microphone/camcorder, join the channel and start peering up */
+      join_chat_channel(ws, table.game.id, {'whatever-you-want-here': 'stuff'});
+    });
+  });
+
+  if (!table.game) {
+    return <></>;
+  }
+
+  const setup_local_media = (callback, errorback) => {
+    if (local_media_stream !== undefined) {  /* ie, if we've already been initialized */
+      if (callback) callback();
+      return; 
+    }
+
+    /* Ask user for permission to use the computers microphone and/or camera, 
+      * attach it to an <audio> or <video> tag if they give us access. */
+    console.log("Requesting access to local audio / video inputs");
+
+    navigator.getUserMedia = (navigator.getUserMedia ||
+      navigator.webkitGetUserMedia ||
+      navigator.mozGetUserMedia ||
+      navigator.msGetUserMedia);
+
+    navigator.mediaDevices.getUserMedia({audio: true, video: false})//, "video": true})
+    .then((stream) => { /* user accepted access to a/v */
+      console.log("Access granted to audio/video");
+      
+      local_media_stream = stream;
+      var local_media = document.querySelector('audio.local');
+      if (!local_media) {
+        console.error(`No local media!`);
+      }
+      if (window.URL) {
+        local_media.srcobject = stream;
+      } else {
+        local_media.src = stream;
+      }
+      if (callback) callback();
+    })
+    .catch((error) => { /* user denied access to a/v */
+      console.error(error);
+      console.log("Access denied for audio/video");
+      window.alert("You chose not to provide access to the microphone!" +
+       "Ketran will have no audio :(");
+      if (errorback) errorback();
+    });
+  };
+
+  const join_chat_channel = (ws, channel, userdata) => {
+    ws.send(JSON.stringify({
+      type: 'join', 
+      config: { channel, userdata }
+    }));
+  }
+
+  const part_chat_channel = (channel) => {
+    ws.send(JSON.stringify({ type: 'part', config: channel }));
+  }
+
+  const audioPeers = [];
+  for (let id in peers) {
+    const peer = peers[id];
+    if (peer.attributes) {
+      audioPeers.push(<audio 
+        ref={el => { 
+          if (!el) {
+            return;
+          }
+      
+          if (peer.extra.srcObject) {
+            el.srcObject = peer.extra.srcObject;
+          } else {
+            el.src = peer.extra.src;
+          }
+        }}
+        key={`Peer-${id}`} className="Remote" {...peer.attributes}/>);
+    }
+  }
+
+  return <div className="MediaAgent">
+    <audio className="local" autoPlay="autoplay" muted={true} controls/>
+    { audioPeers }
+  </div>;
+}
+
+const MediaControl = ({table, color }) => {
+  const [mute, setMute] = useState(false);
+  const [mic, setMic] = useState(true);
+  const [noAudio, setNoAudio] = useState(false);
+  const [rtc, setRtc] = useState(undefined);
+
+  if (!table || !table.game) {
+    return <></>;
+  }
+
+ /*
+  const player = table.game.player;
+  if (!player) {
+    return <></>;
+  }
+*/
+  if (table.game.players[color].status !== 'Active') {
+    return <div className="MediaControl"/>;
+  }
+
+  const isSelf = table.game.color === color;
+
+  if (!rtc) {
+//    setRtc(RTC({ id: table.game.id }));
+  }
+
+  const toggleMic = (event) => {
+    setMic(!mic);
+    event.stopPropagation();
+  }
+
+  const toggleMute = (event) => {
+    setMute(!mute);
+    event.stopPropagation();
+  }
+
+  return (<div className="MediaControl">
+    { isSelf && <div onClick={toggleMic}>
+      { mic && <MicOff color={noAudio ? 'disabled' : 'primary'}/> }
+      { !mic && <Mic color={noAudio ? 'disabled' : 'primary'}/> }
+    </div> }
+    <div onClick={toggleMute}>
+      { mute && <VolumeOff color={noAudio ? 'disabled' : 'primary'}/> }
+      { !mute && <VolumeUp color={noAudio ? 'disabled' : 'primary'}/> }
+    </div>
+  </div>);
+};
+
+export { MediaControl, MediaAgent };

client/src/Table.css

@@ -260,14 +260,6 @@
   padding: 0;
 }
 
-.PlayerSelector .PlayerEntry {
-  flex: 1 1 0px;
-  align-items: center;
-  display: inline-flex;
-  flex-direction: row;
-  min-width: 10em;
-}
-
 .PlayerSelector .MuiTypography-body1 {
   font-size: 0.8rem;
 /*  white-space: nowrap;*/
@@ -282,6 +274,15 @@
   border: 1px solid rgba(0,0,0,0);
   border-radius: 0.5em;
   min-width: 10em;
+  justify-content: space-between;
+}
+
+.PlayerSelector .PlayerEntry {
+  flex: 1 1 0px;
+  align-items: center;
+  display: inline-flex;
+  flex-direction: row;
+  min-width: 10em;
 }
 
 .Players .PlayerEntry[data-selectable=true]:hover {
@@ -293,6 +294,7 @@
   background-color: rgba(255, 255, 0, 0.5);
 }
 
+
 .Players .PlayerToggle {
   min-width: 5em;
   display: inline-flex;

client/src/Table.js

@@ -7,7 +7,6 @@ import TextField from '@material-ui/core/TextField';
 import List from '@material-ui/core/List';
 import Board from './Board.js';
 import Trade from './Trade.js';
-import { assetsPath, base, getPlayerName, gamesPath } from './Common.js';
 import PlayerColor from './PlayerColor.js';
 import Dice from './Dice.js';
 import Resource from './Resource.js';
@@ -20,6 +19,8 @@ import 'moment-timezone';
 import Activities from './Activities.js';
 import Placard from './Placard.js';
 import PlayersStatus from './PlayersStatus.js';
+import { MediaAgent, MediaControl } from './MediaControl.js';
+import { base, assetsPath, getPlayerName, gamesPath } from './Common.js';
 
 /* Start of withRouter polyfill */
 // https://reactrouter.com/docs/en/v6/faq#what-happened-to-withrouter-i-need-it
@@ -300,6 +301,7 @@ const Players = ({ table }) => {
         onClick={() => { inLobby && selectable && toggleSelected(color) }} 
         key={`player-${color}`}>
         <PlayerColor color={color}/>{name}
+        <MediaControl color={color} table={table}/>
       </div>
     ));
   }
@@ -672,6 +674,10 @@ class Table extends React.Component {
   resetKeepAlive(isDead) {
     if (isDead) {
       console.log(`Short circuiting keep-alive`);
+      if (this.ws) {
+        this.ws.close();
+        delete this.ws;
+      }
     } else {
       console.log(`${this.game.name} Resetting keep-alive. Last ping: ${(Date.now() - this.lastPing) / 1000}`);
     }
@@ -688,6 +694,7 @@ class Table extends React.Component {
       this.setState({ noNetwork: true });
       if (this.ws) {
         this.ws.close();
+        delete this.ws;
       }
       this.connectWebSocket();
     }, isDead ? 3000 : 10000);
@@ -705,14 +712,16 @@ class Table extends React.Component {
       this.resetKeepAlive(true);
       return;
     }
-
+    if (this.ws) {
+      return;
+    }
     let loc = window.location, new_uri;
     if (loc.protocol === "https:") {
       new_uri = "wss";
     } else {
       new_uri = "ws";
     }
-    new_uri = `${new_uri}://${loc.host}${base}/api/v1/games/ws/${this.game.id}`;
+    new_uri = `${new_uri}://${loc.host}${base}/api/v1/games/ws/${this.game.id}/`;
     console.log(`Attempting WebSocket connection to ${new_uri}`);
 
     this.ws = new WebSocket(new_uri);
@@ -743,7 +752,6 @@ class Table extends React.Component {
           this.ws.send(JSON.stringify({ type: 'pong', timestamp: data.ping }));
           break;
         default:
-          console.log(`Unknown event type: ${data.type}`);
           break;
       }
     });
@@ -765,7 +773,6 @@ class Table extends React.Component {
       this.ws.send(JSON.stringify({ type: 'game-update' }));
       this.resetKeepAlive();
     });
-  
   }
 
   componentDidMount() {
@@ -925,7 +932,8 @@ class Table extends React.Component {
       return <></>;
     }
 
-    return (
+    return (<>
+      <MediaAgent table={this}/>
       <div className="Table">
         { this.state.loading > 0 && <CircularProgress className='Loading'/> } 
 
@@ -1018,7 +1026,7 @@ class Table extends React.Component {
         { this.state.error && <Paper onClick={() => this.setState({ error: undefined })} className="Error"><div>{this.state.error}</div></Paper> }
 
       </div>
-    );
+    </>);
   }
 }
 export default withRouter(props => <Table {...props}/>);

client/src/setupProxy.js

@@ -2,7 +2,7 @@ const { createProxyMiddleware } = require('http-proxy-middleware');
 
 module.exports = function(app) {
   const base = process.env.PUBLIC_URL;
-  console.log('http-proxy-middleware');
+  console.log(`http-proxy-middleware ${base}`);
   app.use(createProxyMiddleware(
     `${base}/api/v1/games/ws`, {
       ws: true,

server/app.js

@@ -12,8 +12,20 @@ const express = require("express"),
   SQLiteStore = require('connect-sqlite3')(session),
   basePath = require("./basepath"),
   app = express(),
-  server = require("http").createServer(app),
+  fs = require('fs');
-  ws = require('express-ws')(app, server);
+
+let server;
+
+if (0) {
+  const key = fs.readFileSync("ssl/server-key.pem", "utf8"),
+    cert = fs.readFileSync("ssl/server-cert.pem", "utf8"),
+    credentials = { key, cert };
+  server = require("https").createServer(credentials, app);
+} else {
+  server = require("http").createServer(app);
+}
+
+const ws = require('express-ws')(app, server);
 
 require("./console-line.js"); /* Monkey-patch console.log with line numbers */
 

server/routes/games.js

@@ -2719,9 +2719,43 @@ const resetDisconnectCheck = (game, req) => {
   //req.disconnectCheck = setTimeout(() => { wsInactive(game, req) }, 20000);
 }
 
+const channels = {},
+  sockets = {};
+
+const part = (ws, channel) => {
+  console.log(`[${ws.id}] part `);
+
+  if (!(channel in ws.channels)) {
+    console.log(`[${ws.id}] ERROR: not in `, channel);
+    return;
+  }
+
+  delete ws.channels[channel];
+  delete channels[channel][id];
+
+  for (id in channels[channel]) {
+    channels[channel][id].send(JSON.stringify({
+      type: 'removePeer',
+      data: {'peer_id': id}
+    }));
+    ws.send(JSON.stringify({
+      type: 'removePeer', 
+      data: {'peer_id': id}
+    }));
+  }
+};
+
 router.ws("/ws/:id", async (ws, req) => {
   const { id } = req.params;
 
+  ws.id = req.session.player_id;
+  ws.channels = {},
+
+  sockets[ws.id] = ws;
+  
+
+  console.log(`Connection for ${ws.id}`);
+
   /* Setup WebSocket event handlers prior to performing any async calls or
    * we may miss the first messages from clients */
   ws.on('error', async (event) => {
@@ -2754,32 +2788,87 @@ router.ws("/ws/:id", async (ws, req) => {
         session.ws = undefined;
       }
     }
+    /* Cleanup any voice channels */
+    for (let channel in ws.channels) {
+      part(channel);
+    }
+    console.log(`[${ws.id}] disconnected`);
+    delete sockets[ws.id];
   });
 
   ws.on('message', async (message) => {
-    /* Ensure the session is loaded prior to the first 'message'
+    const data = JSON.parse(message);
-     * being processed */
+
+    switch (data.type) {
+    case 'join': {
+      const { channel, userdata } = data.config;
+      console.log(`[${ws.id}] join `, channel);
+
+      if (channel in ws.channels) {
+        console.log(`[${ws.id}] ERROR: already joined `, channel);
+        return;
+      }
+
+      if (!(channel in channels)) {
+        channels[channel] = {};
+      }
+
+      for (let id in channels[channel]) {
+        channels[channel][id].send(JSON.stringify({
+          type: 'addPeer', 
+          data: { 'peer_id': id, 'should_create_offer': false }
+        }));
+        ws.send(JSON.stringify({
+          type: 'addPeer', 
+          data: {'peer_id': id, 'should_create_offer': true}
+        }));
+      }
+
+      channels[channel][ws.id] = ws;
+      ws.channels[channel] = channel;
+    } break;
+    case 'part':
+      part(ws, data.part);
+      break;
+    case 'relayICECandidate': {
+      const { peer_id, ice_candidate } = data.config;
+      console.log(`[${ws.id}] relaying ICE candidate to [${peer_id}] `, ice_candidate);
+  
+      if (peer_id in sockets) {
+        sockets[peer_id].send(JSON.stringify({
+          type: 'iceCandidate', 
+          data: {'peer_id': ws.id, 'ice_candidate': ice_candidate }
+        }));
+      }
+    } break;
+    case 'relaySessionDescription': {
+      const { peer_id, session_description } = data.config;
+      console.log(`[${ws.id}] relaying session description to [${peer_id}] `, session_description);
+      if (peer_id in sockets) {
+        sockets[peer_id].send(JSON.stringify({
+          type: 'sessionDescription', 
+          data: {'peer_id': ws.id, 'session_description': session_description }
+        }));
+      }
+    } break;
+    case 'pong': {
       const game = await loadGame(id);
       if (!game) {
         console.error(`Unable to load/create new game for WS request.`);
         return;
       }
-    const session = getSession(game, req.session);
-    try {
-      const data = JSON.parse(message);
-      switch (data.type) {
-      case 'pong':
-        // console.log(`Latency for ${session.name ? session.name : 'Unammed'} is ${Date.now() - data.timestamp}`);
       resetDisconnectCheck(game, req);
-        break;
+    } break;
-      case 'game-update':
+    case 'game-update': {
+      const game = await loadGame(id);
+      if (!game) {
+        console.error(`Unable to load/create new game for WS request.`);
+        return;
+      }
       console.log(`Player ${session.name ? session.name : 'Unnamed'} requested a game update.`);
       resetDisconnectCheck(game, req);
       sendGame(req, undefined, game, undefined, ws);
-        break;
+    } break;
-      }
-    } catch (error) {
-      console.error(error);
     }
   });