From ec557e6926e6288b77558dc51101d8b14d74a3ac Mon Sep 17 00:00:00 2001
From: James Ketrenos <james_eikona@ketrenos.com>
Date: Sun, 20 Mar 2022 11:01:06 -0700
Subject: [PATCH] Allow non-media players to connect to other players

Signed-off-by: James Ketrenos <james_eikona@ketrenos.com>
---
 client/src/MediaControl.js | 156 +++++++++++++++++++++++--------------
 server/routes/games.js     |  33 +++++---
 2 files changed, 120 insertions(+), 69 deletions(-)

diff --git a/client/src/MediaControl.js b/client/src/MediaControl.js
index b12ddd0..707f1d3 100644
--- a/client/src/MediaControl.js
+++ b/client/src/MediaControl.js
@@ -78,34 +78,16 @@ const MediaAgent = ({setPeers}) => {
           return;
         }
       }
-      const connection = new RTCPeerConnection({
-        configuration: {
-          offerToReceiveAudio: true,
-          offerToReceiveVideo: true
-        },
-        iceServers: [ {
-            urls: "turns:ketrenos.com:5349",
-            username: "ketra",
-            credential: "ketran"
-          },
-        /*
-        {
-          urls: "turn:numb.viagenie.ca",
-          username: "james_viagenie@ketrenos.com",
-          credential: "1!viagenie"
-        } 
-        */
-        ]
-      });
-
-      /* Even if reviving, allocate a new Object so <MediaControl> will
-       * have its peer state change and trigger an update from 
-       * <PlayerList> */
+       /* Even if reviving, allocate a new Object so <MediaControl> will
+      * have its peer state change and trigger an update from 
+      * <PlayerList> */
       const peer = {
         name: peer_id,
-        connection,
+        hasAudio: config.hasAudio,
+        hasVideo: config.hasVideo,
         attributes: {},
       };
+    
       if (peer_id in peers) {
         peer.muted = peers[peer_id].muted;
         peer.videoOn = peers[peer_id].videoOn;
@@ -119,8 +101,35 @@ const MediaAgent = ({setPeers}) => {
       console.log(`media-agent - addPeer - remote`, peers);
       setPeers(Object.assign({}, peers));
       
+      const connection = new RTCPeerConnection({
+        configuration: {
+          offerToReceiveAudio: true,
+          offerToReceiveVideo: true
+        },
+        iceServers: [{
+          urls: "turns:ketrenos.com:5349",
+          username: "ketra",
+          credential: "ketran"
+        },
+          /*
+          {
+            urls: "turn:numb.viagenie.ca",
+            username: "james_viagenie@ketrenos.com",
+            credential: "1!viagenie"
+          } 
+          */
+        ]
+      });
+      peer.connection = connection;
+
       connection.addEventListener('connectionstatechange', (event) => {
-        console.log(`media-agent - connectionstatechange - `, connection.connectionState, event);
+        console.log(`media-agent - connectionstatechange - `,
+          connection.connectionState, event);
+      });
+
+      connection.addEventListener('negotiationneeded', (event) => {
+        console.log(`media-agent - negotiationneeded - `,
+          connection.connectionState, event);
       });
       
       connection.addEventListener('icecandidateerror', (event) => {
@@ -163,7 +172,7 @@ const MediaAgent = ({setPeers}) => {
       connection.ontrack = e => refOnTrack.current(e);
 
       /* Add our local stream */
-      connection.addStream(stream);
+      connection.addStream(stream.media);
   
       /* Only one side of the peer connection should create the
        * offer, the signaling server picks one to be the offerer. 
@@ -172,10 +181,12 @@ const MediaAgent = ({setPeers}) => {
        * to us
        */
       if (config.should_create_offer) {
-        if (debug) console.log(`media-agent - Creating RTC offer to ${peer_id}`);
+        if (debug) console.log(`media-agent - Creating RTC offer to ` +
+          `${peer_id}`);
         return connection.createOffer()
         .then((local_description) => { 
-          if (debug) console.log(`media-agent - Local offer description is: `, local_description);
+          if (debug) console.log(`media-agent - Local offer ` +
+            `description is: `, local_description);
           return connection.setLocalDescription(local_description)
           .then(() => { 
             sendMessage({
@@ -185,7 +196,8 @@ const MediaAgent = ({setPeers}) => {
                 'session_description': local_description
               }
             });
-            if (debug) console.log(`media-agent - Offer setLocalDescription succeeded`);
+            if (debug) console.log(`media-agent - Offer ` +
+              `setLocalDescription succeeded`);
           })
           .catch((error) => {
             console.error(`media-agent - Offer setLocalDescription failed!`); 
@@ -200,17 +212,21 @@ const MediaAgent = ({setPeers}) => {
     const sessionDescription = ({ peer_id, session_description }) => {
       const peer = peers[peer_id];
       if (!peer) {
-        console.error(`media-agent - sessionDescription - No peer for ${peer_id}`);
+        console.error(`media-agent - sessionDescription - ` +
+          `No peer for ${peer_id}`);
         return;
       }
       const { connection } = peer;
       const desc = new RTCSessionDescription(session_description);
       return connection.setRemoteDescription(desc, () => {
-        if (debug) console.log(`media-agent - sessionDescription - setRemoteDescription succeeded`);
+        if (debug) console.log(`media-agent - sessionDescription - ` +
+          `setRemoteDescription succeeded`);
         if (session_description.type === "offer") {
-          if (debug) console.log(`media-agent - sessionDescription - Creating answer`);
+          if (debug) console.log(`media-agent - sessionDescription - ` +
+            `Creating answer`);
           connection.createAnswer((local_description) => {
-            if (debug) console.log(`media-agent - sessionDescription - Answer description is: `, local_description);
+            if (debug) console.log(`media-agent - sessionDescription - ` +
+              `Answer description is: `, local_description);
             connection.setLocalDescription(local_description, () => { 
               sendMessage({
                 type: 'relaySessionDescription',
@@ -219,21 +235,25 @@ const MediaAgent = ({setPeers}) => {
                   session_description: local_description
                 }
               });
-              if (debug) console.log(`media-agent - sessionDescription - Answer setLocalDescription succeeded`);
+              if (debug) console.log(`media-agent - sessionDescription ` +
+                `- Answer setLocalDescription succeeded`);
             }, () => {
-              console.error(`media-agent - sessionDescription - Answer setLocalDescription failed!`);
+              console.error(`media-agent - sessionDescription - ` +
+                `Answer setLocalDescription failed!`);
             });
           }, (error) => {
             console.error(error);
           });
         }
       }, (error) => {
-        console.log(`media-agent - sessionDescription - setRemoteDescription error: `, error);
+        console.log(`media-agent - sessionDescription - ` +
+        `setRemoteDescription error: `, error);
       });
     };
   
     const removePeer = ({peer_id}) => {
-      console.log(`media-agent - removePeer - Signaling server said to remove peer ${peer_id}`);
+      console.log(`media-agent - removePeer - Signaling server said to ` +
+        `remove peer ${peer_id}`);
       if (peer_id in peers) {
         if (peers[peer_id].connection) {
           peers[peer_id].connection.close();
@@ -256,12 +276,14 @@ const MediaAgent = ({setPeers}) => {
       */
       const peer = peers[peer_id];
       if (!peer) {
-        console.error(`media-agent - iceCandidate - No peer for ${peer_id}`, peers);
+        console.error(`media-agent - iceCandidate - No peer for ` +
+          `${peer_id}`, peers);
         return;
       }
       peer.connection.addIceCandidate(new RTCIceCandidate(candidate))
       .then(() => {
-        if (debug) console.log(`media-agent - iceCandidate - Successfully added Ice Candidate for ${peer_id}`);
+        if (debug) console.log(`media-agent - iceCandidate - ` +
+          `Successfully added Ice Candidate for ${peer_id}`);
       })
       .catch((error) => {
         console.error(error, peer, candidate);
@@ -269,7 +291,8 @@ const MediaAgent = ({setPeers}) => {
     };
  
     const data = JSON.parse(event.data);
-    if (data.type in [ 'addPeer', 'removePeer', 'iceCandidate', 'sessionDescription' ]) {
+    if (data.type in [ 'addPeer', 'removePeer', 
+      'iceCandidate', 'sessionDescription' ]) {
       console.log(`media-agent - message - ${data.type}`, peers);
     }
     switch (data.type) {
@@ -329,7 +352,13 @@ const MediaAgent = ({setPeers}) => {
     console.log(`media-control - WebSocket or Stream changed`);
 
     const join = () => {
-      sendMessage({ type: 'join' });
+      sendMessage({ 
+        type: 'join',
+        config: {
+          hasAudio: stream.audio,
+          hasVideo: stream.video
+        }
+      });
     }
     
     if (ws && stream) {
@@ -359,9 +388,11 @@ const MediaAgent = ({setPeers}) => {
           local: true,
           muted: true,
           videoOn: false,
+          hasVideo: stream.video,
+          hasAudio: stream.audio,
           attributes: {
             local: true,
-            srcObject: stream
+            srcObject: stream.media
           }
         };
       }
@@ -390,11 +421,12 @@ const MediaAgent = ({setPeers}) => {
     const setup_local_media = () => { 
       /* Ask user for permission to use the computers microphone and/or camera, 
         * attach it to an <audio> or <video> tag if they give us access. */
-      console.log("media-agent - Requesting access to local audio / video inputs");
+      console.log(`media-agent - Requesting access to local ` +
+        `audio / video inputs`);
   
       /* See Dummy Tracks for more ideas...
-      https://blog.mozilla.org/webrtc/warm-up-with-replacetrack/
-      */
+       * https://blog.mozilla.org/webrtc/warm-up-with-replacetrack/
+       */
       navigator.getUserMedia = (navigator.getUserMedia ||
         navigator.webkitGetUserMedia ||
         navigator.mozGetUserMedia ||
@@ -453,7 +485,7 @@ const MediaAgent = ({setPeers}) => {
               }
             });
           });
-          return context.media;
+          return context;
         }
 
         const black = ({ width = 640, height = 480 } = {}) => {
@@ -489,23 +521,26 @@ const MediaAgent = ({setPeers}) => {
               enabled: true
             });
           }
-          return new MediaStream([context.media, black()]);
+          context.media = new MediaStream([context.media, black()]);
+          return context;
         }
 
-        return new MediaStream([black(), silence()]);
+        context.media = new MediaStream([black(), silence()]);
+        return context;
       });
     };
   
     let abort = false;
     if (!stream) {
-      if (debug) console.log(`media-agent - WebSocket open request. Attempting to create local media.`);
-      setup_local_media().then((media) => {
+      if (debug) console.log(`media-agent - WebSocket open request. ` +
+        `Attempting to create local media.`);
+      setup_local_media().then((context) => {
         /* once the user has given us access to their
          * microphone/camcorder, join the channel and start peering up */
         if (abort) {
           console.log(`media-agent - aborting setting local media`);
         } else {
-          setStream(media);
+          setStream(context);
         }
       }).catch((error) => { /* user denied access to a/v */
         console.error(error);
@@ -565,7 +600,7 @@ const MediaControl = ({isSelf, peer, className}) => {
     if (media.attributes.srcObject) {
       console.log(`media-control - audio enable - ${peer.name}:${!muted}`);
       media.attributes.srcObject.getAudioTracks().forEach((track) => {
-        track.enabled = !muted;
+        track.enabled = media.hasAudio && !muted;
       });
     }
   }); /* run after every render to hit when ontrack has received and set
@@ -578,28 +613,29 @@ const MediaControl = ({isSelf, peer, className}) => {
     if (media.attributes.srcObject) {
       console.log(`media-control - video enable - ${peer.name}:${videoOn}`);
       media.attributes.srcObject.getVideoTracks().forEach((track) => {
-        track.enabled = videoOn;
+        track.enabled = media.hasVideo && videoOn;
       });
     }
   }); /* run after every render  to hit when ontrack has received and set
        * the stream //, [media, videoOn]); */
 
   const isValid = media && !media.dead,
-    color = isValid ? 'primary' : 'disabled';
+    colorAudio = (isValid && media.hasAudio) ? 'primary' : 'disabled',
+    colorVideo = (isValid && media.hasVideo) ? 'primary' : 'disabled';
 
   return <div className={`MediaControl ${className}`}>
     <div className="Controls" >
      { isSelf && <div onClick={toggleMute}>
-        { muted && <MicOff color={color}/> }
-        { !muted && <Mic color={color}/> }
+        { muted && <MicOff color={colorAudio}/> }
+        {!muted && <Mic color={colorAudio}/> }
       </div> }
       { !isSelf && <div onClick={toggleMute}>
-        { muted && <VolumeOff color={color}/> }
-        { !muted && <VolumeUp color={color}/> }
+        {muted && <VolumeOff color={colorAudio}/> }
+        {!muted && <VolumeUp color={colorAudio}/> }
       </div> }
       <div onClick={toggleVideo}>
-        { !videoOn && <VideocamOff color={color}/> }
-        { videoOn && <Videocam color={color}/> }
+        { !videoOn && <VideocamOff color={colorVideo}/> }
+        {videoOn && <Videocam color={colorVideo}/> }
       </div>
     </div>
     { isValid && <Video className="Video"
diff --git a/server/routes/games.js b/server/routes/games.js
index fd1ca86..c2e13e9 100755
--- a/server/routes/games.js
+++ b/server/routes/games.js
@@ -2961,7 +2961,7 @@ const resetDisconnectCheck = (game, req) => {
   //req.disconnectCheck = setTimeout(() => { wsInactive(game, req) }, 20000);
 }
 
-const join = (peers, session) => {
+const join = (peers, session, { hasVideo, hasAudio }) => {
   const ws = session.ws;
 
   if (!session.name) {
@@ -2978,19 +2978,34 @@ const join = (peers, session) => {
   }
 
   for (let peer in peers) {
-    peers[peer].send(JSON.stringify({
+    /* Add this caller to all peers */
+    peers[peer].ws.send(JSON.stringify({
       type: 'addPeer', 
-      data: { 'peer_id': session.name, 'should_create_offer': false }
+      data: {
+        peer_id: session.name, 
+        should_create_offer: false,
+        hasAudio, hasVideo
+      }
     }));
 
+    /* Add each other peer to the caller */
     ws.send(JSON.stringify({
       type: 'addPeer', 
-      data: {'peer_id': peer, 'should_create_offer': true}
+      data: {
+        peer_id: peer, 
+        should_create_offer: true,
+        hasAudio: peers[peer].hasAudio,
+        hasVideo: peers[peer].hasVideo
+      }
     }));
   }
 
   /* Add this user as a peer connected to this WebSocket */
-  peers[session.name] = ws;
+  peers[session.name] = {
+    ws,
+    hasAudio,
+    hasVideo
+  };
 };
 
 const part = (peers, session) => {
@@ -3014,7 +3029,7 @@ const part = (peers, session) => {
   /* Remove this peer from all other peers, and remove each
    * peer from this peer */
   for (let peer in peers) {
-    peers[peer].send(JSON.stringify({
+    peers[peer].ws.send(JSON.stringify({
       type: 'removePeer',
       data: {'peer_id': session.name}
     }));
@@ -3533,7 +3548,7 @@ router.ws("/ws/:id", async (ws, req) => {
 
     switch (data.type) {
     case 'join':
-      join(audio[id], session);
+      join(audio[id], session, data.config);
       break;
 
     case 'part':
@@ -3556,7 +3571,7 @@ router.ws("/ws/:id", async (ws, req) => {
       });
 
       if (peer_id in audio[id]) {
-        audio[id][peer_id].send(message);
+        audio[id][peer_id].ws.send(message);
       }
     } break;
 
@@ -3573,7 +3588,7 @@ router.ws("/ws/:id", async (ws, req) => {
         data: {'peer_id': getName(session), 'session_description': session_description }
       });
       if (peer_id in audio[id]) {
-        audio[id][peer_id].send(message);
+        audio[id][peer_id].ws.send(message);
       }
     } break;