Audio Video almost working; need to merge "users" and "peers"

client/src/App.tsx

@@ -23,17 +23,19 @@ const Lobby: React.FC<LobbyProps> = (props: LobbyProps) => {
   const [global, setGlobal] = useState<GlobalContextType>({
     connected: false,
     ws: undefined,
+    sessionId: sessionId,
     name: "",
     chat: [],
   });
 
-  useEffect(() => {
-    console.log(global);
-  }, [global]);
-
   const onWsMessage = (event: MessageEvent) => {
     const data = JSON.parse(event.data);
     switch (data.type) {
+      case "update":
+        if ("name" in data) {
+          setName(data.name);
+        }
+        break;
       case "error":
         setError(data.error);
         break;

client/src/GlobalContext.tsx

@@ -4,6 +4,7 @@ interface GlobalContextType {
   connected: boolean;
   ws?: WebSocket;
   name?: string;
+  sessionId?: string;
   chat?: any[];
   [key: string]: any;
 }
@@ -12,6 +13,7 @@ const GlobalContext = createContext<GlobalContextType>({
   connected: false,
   ws: undefined,
   name: "",
+  sessionId: undefined,
   chat: []
 });
 

client/src/MediaControl.tsx

@@ -14,7 +14,8 @@ const debug = true;
 
 // Types for peer and track context
 interface Peer {
-  name: string;
+  sessionId: string;
+  peerName: string;
   hasAudio: boolean;
   hasVideo: boolean;
   attributes: Record<string, any>;
@@ -33,6 +34,7 @@ interface TrackContext {
 
 interface AddPeerConfig {
   peer_id: string;
+  peer_name: string;
   hasAudio: boolean;
   hasVideo: boolean;
   should_create_offer?: boolean;
@@ -85,7 +87,7 @@ type MediaAgentProps = {
 
 const MediaAgent = (props: MediaAgentProps) => {
   const { setPeers } = props;
-  const { name, ws } = useContext(GlobalContext);
+  const { name, ws, sessionId } = useContext(GlobalContext);
   const [peers] = useState<Record<string, Peer>>({});
   const [track, setTrack] = useState<TrackContext | undefined>(undefined);
   const ignore = useRef(false);
@@ -127,7 +129,7 @@ const MediaAgent = (props: MediaAgentProps) => {
           console.log(`media-agent - No local media track`);
           return;
         }
-        const peer_id = config.peer_id;
+        const { peer_id, peer_name } = config;
         if (peer_id in peers) {
           if (!peers[peer_id].dead) {
             /* This is normal when peers are added by other connecting
@@ -140,7 +142,8 @@ const MediaAgent = (props: MediaAgentProps) => {
          * have its peer state change and trigger an update from
          * <PlayerList> */
         const peer: Peer = {
-          name: peer_id,
+          sessionId: peer_id,
+          peerName: peer_name,
           hasAudio: config.hasAudio,
           hasVideo: config.hasVideo,
           attributes: {},
@@ -442,15 +445,16 @@ const MediaAgent = (props: MediaAgentProps) => {
   }, [ws, track, peers, setPeers, sendMessage]);
 
   useEffect(() => {
-    if (!name) {
+    if (!sessionId) {
       return;
     }
     let update = false;
     if (track) {
-      if (!(name in peers)) {
+      if (!(sessionId in peers)) {
         update = true;
-        peers[name] = {
+        peers[sessionId] = {
-          name: name,
+          peerName: name || "Unknown",
+          sessionId: sessionId,
           local: true,
           muted: true,
           videoOn: false,
@@ -468,7 +472,7 @@ const MediaAgent = (props: MediaAgentProps) => {
     /* Renaming the local connection requires the peer to be deleted
      * and re-established with the signaling server */
     for (let key in peers) {
-      if (peers[key].local && key !== name) {
+      if (peers[key].local && key !== sessionId) {
         delete peers[key];
         update = true;
       }
@@ -485,21 +489,27 @@ const MediaAgent = (props: MediaAgentProps) => {
       return;
     }
 
+    type setup_local_media_props = {
+      audio?: boolean;
+      video?: boolean;
+    }
     const setup_local_media = async (
-      options: { audio?: boolean; video?: boolean } = { audio: true, video: true }
+      props?: setup_local_media_props
     ): Promise<TrackContext> => {
+      const { audio = true, video = true } = props ?? {};
       // Ask user for permission to use the computers microphone and/or camera
       console.log(
-        `media-agent - Requesting access to local audio: ${!!options.audio} / video: ${!!options.video} inputs`
+        `media-agent - Requesting access to local audio: ${audio} / video: ${video} inputs`
       );
       try {
         const media = await navigator.mediaDevices.getUserMedia({
-          audio: !!options.audio,
+          audio,
-          video: !!options.video,
+          video,
         });
-        sendMessage({ type: "media_status", video: !!options.video, audio: !!options.audio });
+        sendMessage({ type: "media_status", video, audio });
         // Optionally apply constraints
-        if (options.video && media.getVideoTracks().length > 0) {
+        if (video && media.getVideoTracks().length > 0) {
+          console.log(`media-agent - Applying video constraints to ${media.getVideoTracks().length} video tracks`);
           media.getVideoTracks().forEach((track) => {
             track.applyConstraints({
               width: { min: 160, max: 320 },
@@ -507,13 +517,13 @@ const MediaAgent = (props: MediaAgentProps) => {
             });
           });
         }
-        return { media, audio: !!options.audio, video: !!options.video };
+        return { media, audio, video };
       } catch (error) {
-        if (options.video) {
+        if (video) {
           console.log(`media-agent - Access to audio and video failed. Trying just audio.`);
           // Try again with only audio if video failed
-          return setup_local_media({ audio: options.audio, video: false });
+          return setup_local_media({ audio, video: false });
-        } else if (options.audio) {
+        } else if (audio) {
           console.log(`media-agent - Access to audio failed.`);
           sendMessage({ type: "media_status", video: false, audio: false });
           // Return a dummy context with no media
@@ -530,14 +540,16 @@ const MediaAgent = (props: MediaAgentProps) => {
       if (debug) console.log(`media-agent - WebSocket open request. ` + `Attempting to create local media.`);
       setup_local_media()
         .then((context) => {
+          console.log(`media-agent - local media setup complete`, context);
           /* once the user has given us access to their
            * microphone/camcorder, join the channel and start peering up */
-          if (ignore.current) {
+          console.log(`media-agent - ignore set to ${ignore.current}`);
-            console.log(`media-agent - aborting setting local media`);
+          // if (ignore.current) {
-          } else {
+          //   console.log(`media-agent - aborting setting local media`);
+          // } else {
             console.log("media-agent - setTrack called with context:", context);
             setTrack(context);
-          }
+          // }
         })
         .catch((error) => {
           /* user denied access to a/v */
@@ -572,8 +584,8 @@ const MediaControl: React.FC<MediaControlProps> = ({ isSelf, peer, className })
   });
 
   useEffect(() => {
-    if (peer && peer.name) {
+    if (peer && peer.peerName) {
-      const el = document.querySelector(`.MediaControl[data-peer="${peer.name}"]`);
+      const el = document.querySelector(`.MediaControl[data-peer="${peer.sessionId}"]`);
       setTarget(el ?? undefined);
     }
   }, [setTarget, peer]);
@@ -594,7 +606,7 @@ const MediaControl: React.FC<MediaControlProps> = ({ isSelf, peer, className })
   console.log(`media-control - render`);
 
   const toggleMute = (event: React.MouseEvent | React.TouchEvent) => {
-    if (debug) console.log(`media-control - toggleMute - ${peer.name}`, !muted);
+    if (debug) console.log(`media-control - toggleMute - ${peer.peerName}`, !muted);
     if (peer) {
       peer.muted = !muted;
       setMuted(peer.muted);
@@ -603,11 +615,11 @@ const MediaControl: React.FC<MediaControlProps> = ({ isSelf, peer, className })
   };
 
   const toggleVideo = (event: React.MouseEvent | React.TouchEvent) => {
-    if (debug) console.log(`media-control - toggleVideo - ${peer.name}`, !videoOn);
+    if (debug) console.log(`media-control - toggleVideo - ${peer.peerName}`, !videoOn);
     if (peer) {
       peer.videoOn = !videoOn;
       if (peer.videoOn && media) {
-        const video = document.querySelector(`video[data-id="${media.name}"]`) as HTMLVideoElement | null;
+        const video = document.querySelector(`video[data-id="${media.peerName}"]`) as HTMLVideoElement | null;
         if (video && typeof video.play === "function") {
           video.play();
         }
@@ -622,7 +634,7 @@ const MediaControl: React.FC<MediaControlProps> = ({ isSelf, peer, className })
       return;
     }
     if (media.attributes.srcObject) {
-      console.log(`media-control - audio enable - ${peer.name}:${!muted}`);
+      console.log(`media-control - audio enable - ${peer.peerName}:${!muted}`);
       (media.attributes.srcObject.getAudioTracks() as MediaStreamTrack[]).forEach((track: MediaStreamTrack) => {
         track.enabled = media.hasAudio && !muted;
       });
@@ -634,7 +646,7 @@ const MediaControl: React.FC<MediaControlProps> = ({ isSelf, peer, className })
       return;
     }
     if (media.attributes.srcObject) {
-      console.log(`media-control - video enable - ${peer.name}:${videoOn}`);
+      console.log(`media-control - video enable - ${peer.peerName}:${videoOn}`);
       (media.attributes.srcObject.getVideoTracks() as MediaStreamTrack[]).forEach((track: MediaStreamTrack) => {
         track.enabled = Boolean(media.hasVideo) && Boolean(videoOn);
       });
@@ -651,9 +663,9 @@ const MediaControl: React.FC<MediaControlProps> = ({ isSelf, peer, className })
   }
 
   return (
-    <Box sx={{ display: 'flex', flexDirection: 'column', alignItems: 'center', border: "3px solid green" }}>
+    <Box sx={{ display: 'flex', flexDirection: 'column', alignItems: 'center', border: "3px solid green", minWidth: '200px', minHeight: '100px' }}>
       <div className={`MediaControlSpacer ${className}`} />
-      <div className={`MediaControl ${className}`} data-peer={peer.name}>
+      <div className={`MediaControl ${className}`} data-peer={peer.sessionId}>
         <div className="Controls">
           {isSelf && (
             <div onTouchStart={toggleMute} onClick={toggleMute}>
@@ -712,7 +724,7 @@ const MediaControl: React.FC<MediaControlProps> = ({ isSelf, peer, className })
             />
             <Video
               className="Video"
-              data-id={media.name}
+              data-id={media.peerName}
               autoPlay={true}
               srcObject={media.attributes.srcObject}
               {...media.attributes}

client/src/UserList.tsx

@@ -6,12 +6,15 @@ import { GlobalContext } from "./GlobalContext";
 import { MediaControl, MediaAgent } from "./MediaControl";
 import Box from "@mui/material/Box";
 
+type User = {
+  name: string;
+  sessionId: string;
+  live: boolean;
+};
+
 const UserList: React.FC = () => {
-  const { ws, name } = useContext(GlobalContext);
+  const { ws, name, sessionId } = useContext(GlobalContext);
-  const [users, setUsers] = useState<Record<string, any>>({});
+  const [users, setUsers] = useState<Record<string, User>>({});
-  const [unselected, setUneslected] = useState<string[]>([]);
-  const [state, setState] = useState<string>('lobby');
-  const [color, setColor] = useState<string | undefined>(undefined);
   const [peers, setPeers] = useState<Record<string, any>>({});
 
   useEffect(() => {
@@ -55,7 +58,6 @@ const UserList: React.FC = () => {
 
   const userElements: JSX.Element[] = [];
 
-  const inLobby = state === 'lobby';
   const sortedUsers: any[] = [];
 
   for (let key in users) {
@@ -88,59 +90,31 @@ const UserList: React.FC = () => {
 
   sortedUsers.sort(sortUsers);
 
-  /* Array of just names... */
-  unselected.sort((A: string, B: string) => {
-    /* active user first */
-    if (A === name) {
-      return -1;
-    }
-    if (B === name) {
-      return +1;
-    }
-    /* Then sort alphabetically */
-    return A.localeCompare(B);
-  });
-
   const videoClass = sortedUsers.length <= 2 ? 'Medium' : 'Small';
   
-  sortedUsers.forEach(user => {
+  sortedUsers.forEach((user : User) => {
     const userName = user.name;
-    const selectable = inLobby && (user.status === 'Not active' || color === user.color);
+    const isSelf = user.sessionId === sessionId;
+    console.log(`User: ${userName}, Is Self: ${isSelf}, hasPeer: ${peers[user.sessionId] ? 'Yes' : 'No'}`);
     userElements.push(
-      <Box sx={{ display: 'flex', flexDirection: 'column', alignItems: 'center', border: "3px solid magenta" }}
+      <Box key={userName} sx={{ display: 'flex', flexDirection: 'column', alignItems: 'center', border: "3px solid magenta" }}
-        data-selectable={selectable}
-        data-selected={user.color === color}
         className="UserEntry"
-        key={`user-${user.color}`}>
+        >
         <div>
           <div className="Name">{userName ? userName : 'Available' }</div>
           { userName && !user.live && <div className="NoNetwork"></div> }
         </div>
-        { userName && user.live && peers[userName] && <MediaControl className={videoClass} peer={peers[userName]} isSelf={user.color === color}/> }
+        { userName && user.live && peers[user.sessionId] && <MediaControl className={videoClass} peer={peers[user.sessionId]} isSelf={isSelf}/> }
-        { !userName && <div></div> }
       </Box>
     );
   });
 
-  const waiting = unselected.map((user) => {
-    return <div className={user === name ? 'Self' : ''} key={user}>
-      <div>{ user }</div>
-      {peers[user] && <MediaControl className={'Small'} peer={peers[user]} isSelf={name === user}/>} 
-    </div>
-  });
-
   return (
     <Paper className={`UserList ${videoClass}`}>
       <MediaAgent setPeers={setPeers}/>
       <List className="UserSelector">
         { userElements }
       </List>
-      { unselected && unselected.length !== 0 && <div className="Unselected">
-        <div>In lobby</div>
-        <div>
-          { waiting }
-        </div>
-      </div> }
     </Paper>
   );
 }

server/main.py

@@ -132,20 +132,22 @@ async def join(
         logger.info(f"{session.short}:{session.name} - Already joined to Audio.")
         return
 
-    logger.info(f"{lobby.short}: -> addPeer - {session.short}:{session.name}")
-
     for peer in lobby.sessions.values():
         if not peer.ws:
             logger.warning(
                 f"{peer.short}:{peer.name} - No WebSocket connection. Skipping."
             )
             continue
+        logger.info(
+            f"{lobby.short}:{peer.name} -> addPeer - {session.short}:{session.name}"
+        )
         # Add this caller to all peers
         await peer.ws.send_json(
             {
                 "type": "addPeer",
                 "data": {
                     "peer_id": session.id,
+                    "peer_name": session.name,
                     "should_create_offer": False,
                     "has_audio": has_audio,
                     "has_video": has_video,
@@ -158,7 +160,8 @@ async def join(
             {
                 "type": "addPeer",
                 "data": {
-                    "peer_id": peer,
+                    "peer_id": peer.id,
+                    "peer_name": peer.name,
                     "should_create_offer": True,
                     "has_audio": peer.has_audio,
                     "has_video": peer.has_video,
@@ -228,7 +231,9 @@ async def websocket_lobby(
     session = getSession(session_id)
     if not session:
         logger.error(f"{short}: Invalid session ID {session_id}")
-        await websocket.send_json({"type": "error", "error": f"Invalid session ID {session_id}"})
+        await websocket.send_json(
+            {"type": "error", "error": f"Invalid session ID {session_id}"}
+        )
         await websocket.close()
         return
     session.ws = websocket
@@ -264,13 +269,18 @@ async def websocket_lobby(
                     logger.info(f"{session.short}: Name set to {session.name}")
                     await websocket.send_json({"type": "update", "name": name})
                 case "list_users":
-                    users = [{"name": s.name, "live": True} for s in sessions.values()]
+                    users = [
+                        {"name": s.name, "live": True, "sessionId": s.id}
+                        for s in sessions.values()
+                    ]
                     await websocket.send_json({"type": "users", "users": users})
 
-                case 'media_status':
+                case "media_status":
                     has_audio = data.get("audio", False)
                     has_video = data.get("video", False)
-                    logger.info(f"{session.short}: <- media-status - audio: {has_audio}, video: {has_video}")
+                    logger.info(
+                        f"{session.short}: <- media-status - audio: {has_audio}, video: {has_video}"
+                    )
                     session.has_audio = has_audio
                     session.has_video = has_video
 
@@ -283,18 +293,19 @@ async def websocket_lobby(
                     await part(lobby, session)
 
                 case "relayICECandidate":
-                    if id not in lobby.sessions:
+                    logger.info(data)
+                    if session.id not in lobby.sessions:
                         logger.error(
                             f"{session.short}:{session.name} <- relayICECandidate - Does not have Audio"
                         )
                         return
 
-                    peer_id = data.peer_id
+                    peer_id = data.get("config", {}).get("peer_id")
-                    candidate = data.candidate
+                    candidate = data.get("config", {}).get("candidate")
 
                     message = {
-                        type: "iceCandidate",
+                        "type": "iceCandidate",
-                        data: {"peer_id": session.id, "candidate": candidate},
+                        "data": {"peer_id": session.id, "candidate": candidate},
                     }
 
                     if peer_id in lobby.sessions:
@@ -312,13 +323,14 @@ async def websocket_lobby(
                         logger.error(
                             f"{session.short}:{session.name} - relaySessionDescription - Does not have Audio"
                         )
-
+                    peer_id = data.get("config", {}).get("peer_id")
-                    peer_id = data.peer_id
+                    session_description = data.get("config", {}).get(
-                    session_description = data.session_description
+                        "session_description"
+                    )
                     message = {
-                        type: "sessionDescription",
+                        "type": "sessionDescription",
-                        data: {
+                        "data": {
-                            "peer_id": session.name,
+                            "peer_id": session.id,
                             "session_description": session_description,
                         },
                     }
@@ -341,6 +353,12 @@ async def websocket_lobby(
 
     except WebSocketDisconnect:
         logger.info(f"WebSocket disconnected for user {session_id}")
+        # Cleanup: remove session from lobby and sessions dict
+        session.ws = None
+        if lobby and session:
+            await part(lobby, session)
+        # if session_id in sessions:
+        #     del sessions[session_id]
 
 
 # Serve static files or proxy to frontend development server
@@ -355,7 +373,6 @@ if PRODUCTION:
 
 
 else:
-
     logger.info(f"Proxying static files to http://static-frontend:3000 at {public_url}")
 
     import ssl
@@ -385,7 +402,8 @@ else:
                 filtered_headers = {
                     k: v
                     for k, v in proxy_resp.headers.items()
-                    if k.lower() not in ["content-encoding", "transfer-encoding", "content-length"]
+                    if k.lower()
+                    not in ["content-encoding", "transfer-encoding", "content-length"]
                 }
                 return Response(
                     content=content,
@@ -405,7 +423,7 @@ else:
     async def websocket_proxy(websocket: StarletteWebSocket):
         logger.info("WebSocket proxy connection established.")
         # Get scheme from websocket.url (should be 'ws' or 'wss')
-        scheme = websocket.url.scheme if hasattr(websocket, 'url') else 'ws'
+        scheme = websocket.url.scheme if hasattr(websocket, "url") else "ws"
         target_url = f"{scheme}://static-frontend:3000/ws"
         await websocket.accept()
         try: