From b26366eb05ff3d71ceaac58c13311d7c4eff3c8a Mon Sep 17 00:00:00 2001 From: James Ketrenos Date: Tue, 26 Aug 2025 17:11:42 -0700 Subject: [PATCH] Fixing bugs --- client/package.json | 2 +- client/src/App.tsx | 325 +++++++------ client/src/Common.ts | 21 +- client/src/GlobalContext.tsx | 27 +- client/src/MediaControl.tsx | 871 ++++++++++++++++++----------------- client/src/UserList.css | 46 +- client/src/UserList.tsx | 189 ++++---- server/main.py | 313 ++++++++----- 8 files changed, 970 insertions(+), 824 deletions(-) diff --git a/client/package.json b/client/package.json index d3c5cfc..f1de2a4 100644 --- a/client/package.json +++ b/client/package.json @@ -17,11 +17,11 @@ "react-moveable": "^0.56.0", "react-router-dom": "^7.8.2", "react-scripts": "5.0.1", + "react-use-websocket": "^4.13.0", "socket.io-client": "^4.8.1", "web-vitals": "^5.1.0" }, "devDependencies": { - "typescript": "^5.4.5", "@types/node": "^20.11.30", "@types/react": "^18.2.70", "@types/react-dom": "^18.2.19", diff --git a/client/src/App.tsx b/client/src/App.tsx index 0c11ae1..e01d065 100644 --- a/client/src/App.tsx +++ b/client/src/App.tsx @@ -1,157 +1,182 @@ -import React, { useState, useEffect, KeyboardEvent, useRef } from "react"; +import React, { useState, useEffect, KeyboardEvent } from "react"; import { Input, Paper, Typography } from "@mui/material"; -import { GlobalContext, GlobalContextType } from "./GlobalContext"; +import { Session } from "./GlobalContext"; import { UserList } from "./UserList"; import "./App.css"; -import { base } from "./Common"; +import { ws_base, base } from "./Common"; import { Box, Button } from "@mui/material"; import { BrowserRouter as Router, Route, Routes, useParams } from "react-router-dom"; +import useWebSocket, { ReadyState } from "react-use-websocket"; console.log(`AI Voice Chat Build: ${process.env.REACT_APP_AI_VOICECHAT_BUILD}`); type LobbyProps = { - lobbyId: string; - sessionId: string; + session: Session; }; -const Lobby: React.FC = (props: LobbyProps) => { - const { lobbyId, sessionId } = props; - const [editName, setEditName] = useState(""); - const [name, setName] = useState(""); - const [ws, setWs] = useState(undefined); - const [error, setError] = useState(null); - const [global, setGlobal] = useState({ - connected: false, - ws: undefined, - sessionId: sessionId, - name: "", - chat: [], - }); - const onWsMessage = (event: MessageEvent) => { - const data = JSON.parse(event.data); +const Lobby: React.FC = (props: LobbyProps) => { + const { session } = props; + const { lobbyName = "default" } = useParams<{ lobbyName: string }>(); + const [lobbyId, setLobbyId] = useState(null); + const [editName, setEditName] = useState(""); + const [error, setError] = useState(null); + const [socketUrl, setSocketUrl] = useState(null); + + const socket = useWebSocket(socketUrl, { + onOpen: () => console.log("WebSocket connection opened."), + onClose: () => console.log("WebSocket connection closed."), + onError: (event) => console.error("WebSocket error observed:", event), + onMessage: (event) => console.log("WebSocket message received:"), + shouldReconnect: (closeEvent) => true, // Will attempt to reconnect on all close events. + reconnectInterval: 3000, + share: true, + }); + const { sendJsonMessage, lastJsonMessage, readyState } = socket; + + useEffect(() => { + if (lobbyId && session) { + setSocketUrl(`${ws_base}/${lobbyId}/${session.id}`); + } + }, [lobbyId, session]); + + useEffect(() => { + if (!lastJsonMessage) { + return; + } + const data: any = lastJsonMessage; switch (data.type) { case "update": if ("name" in data) { - setName(data.name); + console.log(`Lobby - name set to ${data.name}`); + session.name = data.name; } break; case "error": + console.error(`Lobby - Server error: ${data.error}`); setError(data.error); break; default: break; } + }, [lastJsonMessage]); + + useEffect(() => { + console.log("WebSocket connection status: ", readyState); + }, [readyState]); + + useEffect(() => { + if (!session || !lobbyName) { + return; + } + const getLobbyId = async (lobbyName: string, session: Session) => { + const res = await fetch(`${base}/api/lobby/${lobbyName}/${session.id}`, { + method: "GET", + cache: "no-cache", + credentials: "same-origin", + headers: { + "Content-Type": "application/json", + }, + }); + + if (res.status >= 400) { + const error = `Unable to connect to AI Voice Chat server! Try refreshing your browser in a few seconds.`; + console.error(error); + setError(error); + } + + const data = await res.json(); + if (data.error) { + console.error(`Lobby - Server error: ${data.error}`); + setError(data.error); + return; + } + + setLobbyId(data.lobby); + }; + + getLobbyId(lobbyName, session); + }, [session, lobbyName, setLobbyId]); + + const setName = (name: string) => { + sendJsonMessage({ + type: "set_name", + name: name, + }); }; - const refWsMessage = useRef(onWsMessage); - - useEffect(() => { - refWsMessage.current = onWsMessage; - }); - - useEffect(() => { - if (!ws) { - return; - } - const cbMessage = (e: MessageEvent) => refWsMessage.current(e); - ws.addEventListener("message", cbMessage); - return () => { - ws.removeEventListener("message", cbMessage); - }; - }, [ws, refWsMessage]); - - // Setup websocket connection on mount (only once) - useEffect(() => { - if (!lobbyId) { - console.log("No lobby ID"); - return; - } - let loc = window.location, - new_uri; - if (loc.protocol === "https:") { - new_uri = "wss"; - } else { - new_uri = "ws"; - } - new_uri = `${new_uri}://${loc.host}${base}/ws/lobby/${lobbyId}`; - const socket = new WebSocket(new_uri); - socket.onopen = () => { - console.log("WebSocket connected"); - setGlobal((g: GlobalContextType) => ({ ...g, connected: true })); - if (name) { - socket.send(JSON.stringify({ type: "set_name", name })); - } - }; - setWs(socket); - setGlobal((g: GlobalContextType) => ({ ...g, ws: socket })); - return () => { - setGlobal((g: GlobalContextType) => ({ ...g, connected: false, ws: undefined })); - if (socket.readyState !== 0) { - socket.close(); - } - }; - // Only run once on mount - // eslint-disable-next-line - }, []); - - // Update global context and send set_name when name changes - useEffect(() => { - if (!ws || !global.connected || global.name === name) { - return; - } - setGlobal((g: GlobalContextType) => ({ ...g, name })); - console.log("Sending set_name", name); - ws.send(JSON.stringify({ type: "set_name", name })); - }, [name, ws, global]); const handleKeyDown = (event: KeyboardEvent): void => { if (event.key === "Enter") { event.preventDefault(); - if (!editName.trim()) { + const newName = editName.trim(); + if (!newName || session?.name === newName) { return; } - setName(editName.trim()); + setName(newName); setEditName(""); } }; return ( - {!global.connected ? ( + {readyState !== ReadyState.OPEN || !session ? (

Connecting to server...

) : ( - - {global.name && } - - {!global.name && ( - - Enter your name to join: - - { - setEditName(e.target.value); - }} - onKeyDown={handleKeyDown} - placeholder="Your name" - /> - - + <> + + + AI Voice Chat Lobby: {lobbyName} + + {session.name && You are logged in as: {session.name}} + {!session.name && ( + + Enter your name to join: + + { + setEditName(e.target.value); + }} + onKeyDown={handleKeyDown} + placeholder="Your name" + /> + + + + )} + + + + {session.name && ( + <> + {session.lobbies.map((lobby: string) => ( + + + + ))} + + {session && socketUrl && } + )} - + )} {error && ( @@ -163,48 +188,56 @@ const Lobby: React.FC = (props: LobbyProps) => { }; const App = () => { - const [sessionId, setSessionId] = useState(undefined); + const [session, setSession] = useState(null); const [error, setError] = useState(null); - const { lobbyId = "default" } = useParams<{ lobbyId: string }>(); useEffect(() => { - console.log(`App - sessionId`, sessionId); - }, [sessionId]); - - useEffect(() => { - if (sessionId) { + if (!session) { return; } - fetch(`${base}/api/lobby`, { - method: "GET", - cache: "no-cache", - credentials: "same-origin", - headers: { - "Content-Type": "application/json", - }, - }) - .then((res) => { - if (res.status >= 400) { - const error = `Unable to connect to AI Voice Chat server! Try refreshing your browser in a few seconds.`; - console.error(error); - setError(error); - } - return res.json(); - }) - .then((data) => { - setSessionId(data.session); - }) - .catch((error) => {}); - }, [sessionId, setSessionId]); + console.log(`App - sessionId`, session.id); + }, [session]); + + useEffect(() => { + if (session) { + return; + } + const getSession = async () => { + const res = await fetch(`${base}/api/session`, { + method: "GET", + cache: "no-cache", + credentials: "same-origin", + headers: { + "Content-Type": "application/json", + }, + }); + + if (res.status >= 400) { + const error = `Unable to connect to AI Voice Chat server! Try refreshing your browser in a few seconds.`; + console.error(error); + setError(error); + return; + } + const data = await res.json(); + if (data.error) { + console.error(`App - Server error: ${data.error}`); + setError(data.error); + return; + } + setSession(data); + }; + + getSession(); + }, [session, setSession]); return ( - {!sessionId &&

Connecting to server...

} - {sessionId && ( + {!session &&

Connecting to server...

} + {session && ( - } path={`${base}/:lobbyId`} /> - } path={`${base}`} /> + } path={`${base}/:lobbyName`} /> + } path={`${base}`} /> )} diff --git a/client/src/Common.ts b/client/src/Common.ts index 1ce4345..30e70ea 100644 --- a/client/src/Common.ts +++ b/client/src/Common.ts @@ -1,18 +1,5 @@ -function debounce void>(fn: T, ms: number) { - let timer: ReturnType; - return function(this: any, ...args: Parameters) { - clearTimeout(timer); - timer = setTimeout(() => { - timer = null as any; - fn.apply(this, args); - }, ms); - }; -} - const base = process.env.PUBLIC_URL || ""; - -const assetsPath = `${base}/assets`; -const gamesPath = `${base}`; - -export { base, debounce, assetsPath, gamesPath }; -export {}; +const ws_base: string = `${window.location.protocol === "https:" ? "wss" : "ws"}://${ + window.location.host +}${base}/ws/lobby`; +export { base, ws_base }; diff --git a/client/src/GlobalContext.tsx b/client/src/GlobalContext.tsx index 9660a11..9e8c03f 100644 --- a/client/src/GlobalContext.tsx +++ b/client/src/GlobalContext.tsx @@ -1,21 +1,6 @@ -import { createContext } from "react"; - -interface GlobalContextType { - connected: boolean; - ws?: WebSocket; - name?: string; - sessionId?: string; - chat?: any[]; - [key: string]: any; -} - -const GlobalContext = createContext({ - connected: false, - ws: undefined, - name: "", - sessionId: undefined, - chat: [] -}); - -export { GlobalContext }; -export type { GlobalContextType }; +type Session = { + id: string; + name: string | null; + lobbies: string[]; +}; +export type { Session }; diff --git a/client/src/MediaControl.tsx b/client/src/MediaControl.tsx index ec8d998..23f0099 100644 --- a/client/src/MediaControl.tsx +++ b/client/src/MediaControl.tsx @@ -1,4 +1,4 @@ -import React, { useState, useEffect, useRef, useCallback, useContext } from "react"; +import React, { useState, useEffect, useRef, useCallback } from "react"; import Moveable from "react-moveable"; import "./MediaControl.css"; import VolumeOff from "@mui/icons-material/VolumeOff"; @@ -7,27 +7,29 @@ import MicOff from "@mui/icons-material/MicOff"; import Mic from "@mui/icons-material/Mic"; import VideocamOff from "@mui/icons-material/VideocamOff"; import Videocam from "@mui/icons-material/Videocam"; -import { GlobalContext } from "./GlobalContext"; import Box from "@mui/material/Box"; +import useWebSocket from "react-use-websocket"; +import { Session } from "./GlobalContext"; const debug = true; // Types for peer and track context interface Peer { - sessionId: string; + session_id: string; peerName: string; - hasAudio: boolean; - hasVideo: boolean; + has_audio: boolean; + has_video: boolean; attributes: Record; muted: boolean; - videoOn: boolean; + video_on: boolean /* Set by client */; local: boolean; dead: boolean; connection?: RTCPeerConnection; } +export type { Peer }; interface TrackContext { - media: MediaStream; + media: MediaStream | null; audio: boolean; video: boolean; } @@ -35,8 +37,8 @@ interface TrackContext { interface AddPeerConfig { peer_id: string; peer_name: string; - hasAudio: boolean; - hasVideo: boolean; + has_audio: boolean; + has_video: boolean; should_create_offer?: boolean; } @@ -82,15 +84,48 @@ const Video: React.FC = ({ srcObject, local, ...props }) => { }; type MediaAgentProps = { + socketUrl: string; + session: Session; + peers: Record; setPeers: (peers: Record) => void; }; const MediaAgent = (props: MediaAgentProps) => { - const { setPeers } = props; - const { name, ws, sessionId } = useContext(GlobalContext); - const [peers] = useState>({}); - const [track, setTrack] = useState(undefined); - const ignore = useRef(false); + const { peers, setPeers, socketUrl, session } = props; + // track: null = no local media, TrackContext = local media + const [track, setTrack] = useState(null); + + const { sendJsonMessage, lastJsonMessage } = useWebSocket(socketUrl, { + share: true, + onError: (err) => { + console.error(err); + }, + onClose: (_event: CloseEvent) => { + if (!session) { + return; + } + console.log(`media-agent - ${session.name} Disconnected from signaling server`); + /* Tear down all of our peer connections and remove all the + * media divs when we disconnect */ + for (let peer_id in peers) { + if (peers[peer_id].local) { + continue; + } + if (peers[peer_id].connection) { + peers[peer_id].connection.close(); + peers[peer_id].connection = undefined; + } + } + + for (let id in peers) { + peers[id].dead = true; + peers[id].connection = undefined; + } + + if (debug) console.log(`media-agent - close`, peers); + setPeers(Object.assign({}, peers)); + }, + }); const onTrack = useCallback( (event: RTCTrackEvent) => { @@ -102,7 +137,6 @@ const MediaAgent = (props: MediaAgentProps) => { Object.assign(peers[peer].attributes, { srcObject: event.streams[0] || event.track, }); - /* Trigger update of MediaControl now that a track is available */ setPeers(Object.assign({}, peers)); } } @@ -111,326 +145,248 @@ const MediaAgent = (props: MediaAgentProps) => { ); const refOnTrack = useRef(onTrack); - const sendMessage = useCallback( - (data: LobbyMessage) => { - if (!ws) { + const addPeer = useCallback( + (config: AddPeerConfig) => { + console.log("media-agent - addPeer - ", { config, peers }); + if (config.peer_id in peers) { + if (!peers[config.peer_id].dead) { + console.log(`media-agent - addPeer - ${config.peer_id} already in peers`); + return; + } + } + const peer: Peer = { + session_id: config.peer_id, + peerName: config.peer_name, + has_audio: config.has_audio, + has_video: config.has_video, + attributes: {}, + muted: false, + video_on: true, + local: false, + dead: false, + }; + if (config.peer_id in peers) { + peer.muted = peers[config.peer_id].muted; + peer.video_on = peers[config.peer_id].video_on; + } + peers[config.peer_id] = peer; + setPeers({ ...peers }); + const connection = new RTCPeerConnection({ + iceServers: [ + { + urls: "turns:ketrenos.com:5349", + username: "ketra", + credential: "ketran", + }, + ], + }); + peer.connection = connection; + connection.addEventListener("connectionstatechange", (event) => { + console.log(`media-agent - connectionstatechange - `, connection.connectionState, event); + }); + connection.addEventListener("negotiationneeded", (event) => { + console.log(`media-agent - negotiationneeded - `, connection.connectionState, event); + }); + connection.addEventListener("icecandidateerror", (event: RTCPeerConnectionIceErrorEvent) => { + if (event.errorCode === 701) { + if (connection.iceGatheringState === "gathering") { + console.log(`media-agent - Unable to reach host: ${event.url}`); + } else { + console.error( + `media-agent - icecandidateerror - `, + event.errorCode, + (event as any).hostcandidate, + event.url, + event.errorText + ); + } + } + }); + connection.onicecandidate = (event: RTCPeerConnectionIceEvent) => { + if (!event.candidate) { + console.log(`media-agent - icecanditate - gathering is complete: ${connection.connectionState}`); + return; + } + sendJsonMessage({ + type: "relayICECandidate", + config: { + peer_id: config.peer_id, + candidate: event.candidate, + }, + }); + }; + connection.ontrack = (e: RTCTrackEvent) => refOnTrack.current(e); + // Only add local tracks if present + if (track && track.media) { + track.media.getTracks().forEach((t) => { + connection.addTrack(t, track.media!); + }); + } + if (config.should_create_offer) { + connection + .createOffer() + .then((local_description) => { + return connection.setLocalDescription(local_description).then(() => { + sendJsonMessage({ + type: "relaySessionDescription", + config: { + peer_id: config.peer_id, + session_description: local_description, + }, + }); + }); + }) + .catch((error) => { + console.error(`media-agent - Offer setLocalDescription failed!`, error); + }); + } + }, + [peers, setPeers, track, sendJsonMessage] + ); + + const sessionDescription = useCallback( + ({ peer_id, session_description }: SessionDescriptionData) => { + const peer = peers[peer_id]; + if (!peer) { + console.error(`media-agent - sessionDescription - No peer for ${peer_id}`); return; } - ws.send(JSON.stringify(data)); - }, - [ws] - ); - - const onWsMessage = useCallback( - (event: MessageEvent) => { - const addPeer = (config: AddPeerConfig) => { - console.log("media-agent - Signaling server said to add peer:", config); - if (!track) { - console.log(`media-agent - No local media track`); - return; - } - const { peer_id, peer_name } = config; - if (peer_id in peers) { - if (!peers[peer_id].dead) { - /* This is normal when peers are added by other connecting - * peers through the signaling server */ - console.log(`media-agent - addPeer - ${peer_id} already in peers`); - return; - } - } - /* Even if reviving, allocate a new Object so will - * have its peer state change and trigger an update from - * */ - const peer: Peer = { - sessionId: peer_id, - peerName: peer_name, - hasAudio: config.hasAudio, - hasVideo: config.hasVideo, - attributes: {}, - muted: false, - videoOn: true, - local: false, - dead: false, - }; - if (peer_id in peers) { - peer.muted = peers[peer_id].muted; - peer.videoOn = peers[peer_id].videoOn; - console.log(`media-agent - addPeer - reviving dead peer ${peer_id}`, peer); - } else { - peer.muted = false; - peer.videoOn = true; - } - peers[peer_id] = peer; - console.log(`media-agent - addPeer - remote`, peers); - setPeers(Object.assign({}, peers)); - // RTCPeerConnection config should be passed directly, not as 'configuration' property - const connection = new RTCPeerConnection({ - iceServers: [ - { - urls: "turns:ketrenos.com:5349", - username: "ketra", - credential: "ketran", - }, - ], - }); - peer.connection = connection; - - connection.addEventListener("connectionstatechange", (event) => { - console.log(`media-agent - connectionstatechange - `, connection.connectionState, event); - }); - - connection.addEventListener("negotiationneeded", (event) => { - console.log(`media-agent - negotiationneeded - `, connection.connectionState, event); - }); - - connection.addEventListener("icecandidateerror", (event: RTCPeerConnectionIceErrorEvent) => { - if (event.errorCode === 701) { - if (connection.iceGatheringState === "gathering") { - console.log(`media-agent - Unable to reach host: ${event.url}`); - } else { - // hostcandidate is deprecated and not always present - console.error( - `media-agent - icecandidateerror - `, - event.errorCode, - (event as any).hostcandidate, - event.url, - event.errorText - ); - } - } - }); - - connection.onicecandidate = (event: RTCPeerConnectionIceEvent) => { - if (!event.candidate) { - console.log(`media-agent - icecanditate - gathering is complete: ${connection.connectionState}`); - return; - } - // If a srflx candidate was found, notify that the STUN server works! - if (event.candidate && event.candidate.type === "srflx") { - console.log("media-agent - The STUN server is reachable!"); - // address is not standard, use candidate.candidate string parsing if needed - } - // If a relay candidate was found, notify that the TURN server works! - if (event.candidate && event.candidate.type === "relay") { - console.log("media-agent - The TURN server is reachable !"); - } - console.log(`media-agent - onicecandidate - `, event.candidate); - sendMessage({ - type: "relayICECandidate", - config: { - peer_id, - candidate: event.candidate, - }, - }); - }; - - connection.ontrack = (e: RTCTrackEvent) => refOnTrack.current(e); - - // Add all tracks from local media - track.media.getTracks().forEach((t) => { - connection.addTrack(t, track.media); - }); - - /* Only one side of the peer connection should create the - * offer, the signaling server picks one to be the offerer. - * The other user will get a 'sessionDescription' event and will - * create an offer, then send back an answer 'sessionDescription' - * to us - */ - if (config.should_create_offer) { - if (debug) console.log(`media-agent - Creating RTC offer to ${peer_id}`); - connection - .createOffer() - .then((local_description) => { - if (debug) console.log(`media-agent - Local offer description is: `, local_description); - return connection.setLocalDescription(local_description).then(() => { - sendMessage({ - type: "relaySessionDescription", - config: { - peer_id, - session_description: local_description, - }, - }); - if (debug) console.log(`media-agent - Offer setLocalDescription succeeded`); - }); - }) - .catch((error) => { - console.error(`media-agent - Offer setLocalDescription failed!`, error); - }); - } - }; - - const sessionDescription = ({ peer_id, session_description }: SessionDescriptionData) => { - const peer = peers[peer_id]; - if (!peer) { - console.error(`media-agent - sessionDescription - No peer for ${peer_id}`); - return; - } - const { connection } = peer; - if (!connection) { - console.error(`media-agent - sessionDescription - No connection for peer ${peer_id}`); - return; - } - const desc = new RTCSessionDescription(session_description); - connection - .setRemoteDescription(desc) - .then(() => { - if (debug) console.log(`media-agent - sessionDescription - setRemoteDescription succeeded`); - if (session_description.type === "offer") { - if (debug) console.log(`media-agent - sessionDescription - Creating answer`); - connection - .createAnswer() - .then((local_description) => { - if (debug) - console.log(`media-agent - sessionDescription - Answer description is: `, local_description); - connection - .setLocalDescription(local_description) - .then(() => { - sendMessage({ - type: "relaySessionDescription", - config: { - peer_id, - session_description: local_description, - }, - }); - if (debug) console.log(`media-agent - sessionDescription - Answer setLocalDescription succeeded`); - }) - .catch(() => { - console.error(`media-agent - sessionDescription - Answer setLocalDescription failed!`); + const { connection } = peer; + if (!connection) { + console.error(`media-agent - sessionDescription - No connection for peer ${peer_id}`); + return; + } + const desc = new RTCSessionDescription(session_description); + connection + .setRemoteDescription(desc) + .then(() => { + if (debug) console.log(`media-agent - sessionDescription - setRemoteDescription succeeded`); + if (session_description.type === "offer") { + if (debug) console.log(`media-agent - sessionDescription - Creating answer`); + connection + .createAnswer() + .then((local_description) => { + if (debug) console.log(`media-agent - sessionDescription - Answer description is: `, local_description); + connection + .setLocalDescription(local_description) + .then(() => { + sendJsonMessage({ + type: "relaySessionDescription", + config: { + peer_id, + session_description: local_description, + }, }); - }) - .catch((error) => { - console.error(error); - }); - } - }) - .catch((error) => { - console.log(`media-agent - sessionDescription - setRemoteDescription error: `, error); - }); - }; - - const removePeer = ({ peer_id }: RemovePeerData) => { - console.log(`media-agent - removePeer - Signaling server said to ` + `remove peer ${peer_id}`); - if (peer_id in peers) { - if (peers[peer_id].connection) { - peers[peer_id].connection.close(); - peers[peer_id].connection = undefined; + if (debug) console.log(`media-agent - sessionDescription - Answer setLocalDescription succeeded`); + }) + .catch(() => { + console.error(`media-agent - sessionDescription - Answer setLocalDescription failed!`); + }); + }) + .catch((error) => { + console.error(error); + }); } - } - - /* To maintain mute/videoOn states, we don't remove the peer but - * instead mark it as dead */ - peers[peer_id].dead = true; - if (debug) console.log(`media-agent - removePeer`, peers); - setPeers(Object.assign({}, peers)); - }; - - const iceCandidate = ({ peer_id, candidate }: IceCandidateData) => { - /** - * The offerer will send a number of ICE Candidate blobs to the - * answerer so they can begin trying to find the best path to one - * another on the net. - */ - const peer = peers[peer_id]; - if (!peer) { - console.error(`media-agent - iceCandidate - No peer for ${peer_id}`, peers); - return; - } - peer.connection - ?.addIceCandidate(new RTCIceCandidate(candidate)) - .then(() => { - if (debug) console.log(`media-agent - iceCandidate - Successfully added Ice Candidate for ${peer_id}`); - }) - .catch((error) => { - console.error(error, peer, candidate); - }); - }; - - const data = JSON.parse(event.data); - if (["addPeer", "removePeer", "iceCandidate", "sessionDescription"].includes(data.type)) { - console.log(`media-agent - message - ${data.type}`, peers); - } - switch (data.type) { - case "addPeer": - addPeer(data.data); - break; - case "removePeer": - removePeer(data.data); - break; - case "iceCandidate": - iceCandidate(data.data); - break; - case "sessionDescription": - sessionDescription(data.data); - break; - default: - break; - } + }) + .catch((error) => { + console.log(`media-agent - sessionDescription - setRemoteDescription error: `, error); + }); }, - [peers, setPeers, track, refOnTrack, sendMessage] + [peers, sendJsonMessage] ); - const refWsMessage = useRef(onWsMessage); - const onWsClose = (_event: CloseEvent) => { - console.log(`media-agent - ${name} Disconnected from signaling server`); - /* Tear down all of our peer connections and remove all the - * media divs when we disconnect */ - for (let peer_id in peers) { - if (peers[peer_id].local) { - continue; + const removePeer = useCallback( + ({ peer_id }: RemovePeerData) => { + console.log(`media-agent - removePeer - Signaling server said to remove peer ${peer_id}`); + if (peer_id in peers) { + if (peers[peer_id].connection) { + peers[peer_id].connection.close(); + peers[peer_id].connection = undefined; + } } - if (peers[peer_id].connection) { - peers[peer_id].connection.close(); - peers[peer_id].connection = undefined; + + /* To maintain mute/videoOn states, we don't remove the peer but + * instead mark it as dead */ + peers[peer_id].dead = true; + if (debug) console.log(`media-agent - removePeer`, peers); + setPeers(Object.assign({}, peers)); + }, + [peers, setPeers] + ); + + const iceCandidate = useCallback( + ({ peer_id, candidate }: IceCandidateData) => { + /** + * The offerer will send a number of ICE Candidate blobs to the + * answerer so they can begin trying to find the best path to one + * another on the net. + */ + const peer = peers[peer_id]; + if (!peer) { + console.error(`media-agent - iceCandidate - No peer for ${peer_id}`, peers); + return; } - } - - for (let id in peers) { - peers[id].dead = true; - peers[id].connection = undefined; - } - - if (debug) console.log(`media-agent - close`, peers); - setPeers(Object.assign({}, peers)); - }; - const refWsClose = useRef(onWsClose); + peer.connection + ?.addIceCandidate(new RTCIceCandidate(candidate)) + .then(() => { + if (debug) console.log(`media-agent - iceCandidate - Successfully added Ice Candidate for ${peer_id}`); + }) + .catch((error) => { + console.error(error, peer, candidate); + }); + }, + [peers] + ); + + useEffect(() => { + if (!lastJsonMessage) { + return; + } + const data: any = lastJsonMessage; + if (!session) { + return; + } + console.log("media-agent - message", data); + if (["addPeer", "removePeer", "iceCandidate", "sessionDescription"].includes(data.type)) { + console.log(`media-agent - message - ${data.type}`, peers); + } + switch (data.type) { + case "addPeer": + addPeer(data.data); + break; + case "removePeer": + removePeer(data.data); + break; + case "iceCandidate": + iceCandidate(data.data); + break; + case "sessionDescription": + sessionDescription(data.data); + break; + default: + break; + } + }, [lastJsonMessage, addPeer, removePeer, iceCandidate, sessionDescription, peers, session]); useEffect(() => { - refWsMessage.current = onWsMessage; - refWsClose.current = onWsClose; refOnTrack.current = onTrack; }); useEffect(() => { - if (!ws) { - return; - } - console.log(`media-control - Binding to WebSocket`); - const cbMessage = (e: MessageEvent) => refWsMessage.current(e); - ws.addEventListener("message", cbMessage); - const cbClose = (e: CloseEvent) => refWsClose.current(e); - ws.addEventListener("close", cbClose); - return () => { - ws.removeEventListener("message", cbMessage); - ws.removeEventListener("close", cbClose); - }; - }, [ws, refWsMessage, refWsClose]); - - useEffect(() => { - console.log(`media-control - WebSocket or Track changed`); + console.log(`media-control - Track changed`, track); const join = () => { - sendMessage({ + sendJsonMessage({ type: "join", data: { - has_audio: track?.audio ?? false, - has_aideo: track?.video ?? false, + has_audio: track && track.audio ? track.audio : false, + has_video: track && track.video ? track.video : false, }, }); }; - if (ws && track) { - console.log(`media-conterol - issuing join request`); + if (track !== undefined) { + console.log(`media-control - issuing join request: `, track); for (let peer in peers) { if (peers[peer].local && peers[peer].dead) { /* Allocate a new Object so will trigger */ @@ -442,24 +398,24 @@ const MediaAgent = (props: MediaAgentProps) => { } join(); } - }, [ws, track, peers, setPeers, sendMessage]); + }, [track, peers, setPeers, sendJsonMessage]); useEffect(() => { - if (!sessionId) { + if (!session) { return; } let update = false; if (track) { - if (!(sessionId in peers)) { + if (!(session.id in peers)) { update = true; - peers[sessionId] = { - peerName: name || "Unknown", - sessionId: sessionId, + peers[session.id] = { + peerName: session.name || "Unknown", + session_id: session.id, local: true, muted: true, - videoOn: false, - hasVideo: track.video, - hasAudio: track.audio, + video_on: false, + has_video: track.video, + has_audio: track.audio, attributes: { local: true, srcObject: track.media, @@ -472,7 +428,7 @@ const MediaAgent = (props: MediaAgentProps) => { /* Renaming the local connection requires the peer to be deleted * and re-established with the signaling server */ for (let key in peers) { - if (peers[key].local && key !== sessionId) { + if (peers[key].local && key !== session.id) { delete peers[key]; update = true; } @@ -482,89 +438,147 @@ const MediaAgent = (props: MediaAgentProps) => { if (debug) console.log(`media-agent - Setting global peers`, peers); setPeers(Object.assign({}, peers)); } - }, [peers, name, setPeers, track]); + }, [peers, setPeers, track, session]); + + const setup_local_media = async (): Promise => { + console.log(`media-agent - Requesting access to local audio / video inputs`); + const context: TrackContext = { media: null, audio: true, video: true }; + + // Try to get user media with fallback logic + while (context.audio || context.video) { + try { + context.media = await navigator.mediaDevices.getUserMedia({ + audio: context.audio, + video: context.video, + }); + break; + } catch (error) { + console.error(`media-agent - Error accessing local media: `, error); + if (context.video) { + console.log(`media-agent - Disabling video and trying again`); + context.video = false; + } else if (context.audio) { + console.log(`media-agent - Disabling audio and trying again`); + context.audio = false; + } else { + console.log(`media-agent - No media available`); + break; + } + } + } + + // Helper function to create a black video track + const createBlackVideoTrack = ({ width = 640, height = 480 } = {}): MediaStreamTrack => { + const canvas = document.createElement("canvas"); + canvas.width = width; + canvas.height = height; + + const ctx = canvas.getContext("2d"); + if (ctx) { + ctx.fillStyle = "black"; + ctx.fillRect(0, 0, width, height); + } + + const stream = canvas.captureStream(30); // 30 FPS + const track = stream.getVideoTracks()[0]; + track.enabled = true; + return track; + }; + + // Helper function to create a silent audio track + const createSilentAudioTrack = (): MediaStreamTrack => { + const audioContext = new AudioContext(); + const oscillator = audioContext.createOscillator(); + const gainNode = audioContext.createGain(); + const destination = audioContext.createMediaStreamDestination(); + + // Set gain to 0 for silence + gainNode.gain.value = 0; + + // Connect: oscillator -> gain -> destination + oscillator.connect(gainNode); + gainNode.connect(destination); + + oscillator.start(); + + const track = destination.stream.getAudioTracks()[0]; + track.enabled = true; + return track; + }; + + // Process the results and create appropriate media stream + const tracks: MediaStreamTrack[] = []; + let hasRealAudio = false; + let hasRealVideo = false; + + if (context.media) { + const audioTracks = context.media.getAudioTracks(); + const videoTracks = context.media.getVideoTracks(); + + if (audioTracks.length > 0) { + tracks.push(audioTracks[0]); + hasRealAudio = true; + console.log("media-agent - Using real audio"); + } + + if (videoTracks.length > 0) { + // Apply video constraints + videoTracks[0].applyConstraints({ + width: { min: 160, max: 320 }, + height: { min: 120, max: 240 }, + }); + tracks.push(videoTracks[0]); + hasRealVideo = true; + console.log("media-agent - Using real video"); + } + } + + // Add synthetic tracks as needed + if (!hasRealAudio) { + tracks.push(createSilentAudioTrack()); + console.log("media-agent - Using synthetic silent audio"); + } + + if (!hasRealVideo) { + tracks.push(createBlackVideoTrack()); + console.log("media-agent - Using synthetic black video"); + } + + // Create final media stream + context.media = new MediaStream(tracks); + + // Update context flags to reflect what we actually have + context.audio = hasRealAudio; + context.video = hasRealVideo; + + const mediaType = + hasRealAudio && hasRealVideo + ? "audio/video" + : hasRealAudio + ? "audio only" + : hasRealVideo + ? "video only" + : "synthetic audio/video"; + + console.log(`media-agent - Final media setup: ${mediaType}`); + + return context; + }; useEffect(() => { - if (!ws || !name) { + if (!session || !session.name) { return; } - type setup_local_media_props = { - audio?: boolean; - video?: boolean; - } - const setup_local_media = async ( - props?: setup_local_media_props - ): Promise => { - const { audio = true, video = true } = props ?? {}; - // Ask user for permission to use the computers microphone and/or camera - console.log( - `media-agent - Requesting access to local audio: ${audio} / video: ${video} inputs` - ); - try { - const media = await navigator.mediaDevices.getUserMedia({ - audio, - video, - }); - sendMessage({ type: "media_status", video, audio }); - // Optionally apply constraints - if (video && media.getVideoTracks().length > 0) { - console.log(`media-agent - Applying video constraints to ${media.getVideoTracks().length} video tracks`); - media.getVideoTracks().forEach((track) => { - track.applyConstraints({ - width: { min: 160, max: 320 }, - height: { min: 120, max: 240 }, - }); - }); - } - return { media, audio, video }; - } catch (error) { - if (video) { - console.log(`media-agent - Access to audio and video failed. Trying just audio.`); - // Try again with only audio if video failed - return setup_local_media({ audio, video: false }); - } else if (audio) { - console.log(`media-agent - Access to audio failed.`); - sendMessage({ type: "media_status", video: false, audio: false }); - // Return a dummy context with no media - return { media: new MediaStream(), audio: false, video: false }; - } else { - // No media requested or available - sendMessage({ type: "media_status", video: false, audio: false }); - return { media: new MediaStream(), audio: false, video: false }; - } - } - }; - - if (!track) { - if (debug) console.log(`media-agent - WebSocket open request. ` + `Attempting to create local media.`); + if (track === null) { setup_local_media() .then((context) => { - console.log(`media-agent - local media setup complete`, context); - /* once the user has given us access to their - * microphone/camcorder, join the channel and start peering up */ - console.log(`media-agent - ignore set to ${ignore.current}`); - // if (ignore.current) { - // console.log(`media-agent - aborting setting local media`); - // } else { - console.log("media-agent - setTrack called with context:", context); - setTrack(context); - // } + sendJsonMessage({ type: "media_status", ...context, media: undefined }); + setTrack(context); }) - .catch((error) => { - /* user denied access to a/v */ - console.error("media-agent - error in setup_local_media", error); - console.log("media-agent - Access denied for audio/video"); - }); + .catch(() => setTrack(null)); } - - return () => { - ignore.current = true; - if (!track) { - console.log(`media-agent - abort media setup!`); - } - }; - }, [ws, track, name, sendMessage]); + }, [track, session, sendJsonMessage]); return <>; }; @@ -579,13 +593,14 @@ const MediaControl: React.FC = ({ isSelf, peer, className }) const [muted, setMuted] = useState(undefined); const [videoOn, setVideoOn] = useState(undefined); const [target, setTarget] = useState(); + const [isValid, setIsValid] = useState(false); const [frame, setFrame] = useState<{ translate: [number, number] }>({ translate: [0, 0], }); useEffect(() => { if (peer && peer.peerName) { - const el = document.querySelector(`.MediaControl[data-peer="${peer.sessionId}"]`); + const el = document.querySelector(`.MediaControl[data-peer="${peer.session_id}"]`); setTarget(el ?? undefined); } }, [setTarget, peer]); @@ -599,12 +614,10 @@ const MediaControl: React.FC = ({ isSelf, peer, className }) return; } setMuted(peer.muted); - setVideoOn(peer.videoOn); + setVideoOn(peer.video_on); setMedia(peer); }, [peer, setMedia, setMuted, setVideoOn]); - console.log(`media-control - render`); - const toggleMute = (event: React.MouseEvent | React.TouchEvent) => { if (debug) console.log(`media-control - toggleMute - ${peer.peerName}`, !muted); if (peer) { @@ -617,14 +630,14 @@ const MediaControl: React.FC = ({ isSelf, peer, className }) const toggleVideo = (event: React.MouseEvent | React.TouchEvent) => { if (debug) console.log(`media-control - toggleVideo - ${peer.peerName}`, !videoOn); if (peer) { - peer.videoOn = !videoOn; - if (peer.videoOn && media) { + peer.video_on = !videoOn; + if (peer.video_on && media) { const video = document.querySelector(`video[data-id="${media.peerName}"]`) as HTMLVideoElement | null; if (video && typeof video.play === "function") { video.play(); } } - setVideoOn(peer.videoOn); + setVideoOn(peer.video_on); } event.stopPropagation(); }; @@ -636,7 +649,7 @@ const MediaControl: React.FC = ({ isSelf, peer, className }) if (media.attributes.srcObject) { console.log(`media-control - audio enable - ${peer.peerName}:${!muted}`); (media.attributes.srcObject.getAudioTracks() as MediaStreamTrack[]).forEach((track: MediaStreamTrack) => { - track.enabled = media.hasAudio && !muted; + track.enabled = media.has_audio && !muted; }); } }); @@ -648,14 +661,22 @@ const MediaControl: React.FC = ({ isSelf, peer, className }) if (media.attributes.srcObject) { console.log(`media-control - video enable - ${peer.peerName}:${videoOn}`); (media.attributes.srcObject.getVideoTracks() as MediaStreamTrack[]).forEach((track: MediaStreamTrack) => { - track.enabled = Boolean(media.hasVideo) && Boolean(videoOn); + track.enabled = Boolean(media.has_video) && Boolean(videoOn); }); } }); + useEffect(() => { + if (!media || !peer || media.dead || !media.attributes || !media.attributes.srcObject) { + setIsValid(false); + console.log(`media-control - no media or peer`); + return; + } + console.log(`media-control - has valid peer`); + setIsValid(true); + }, [media, peer, setIsValid]); - const isValid = media && !media.dead, - colorAudio = isValid && media.hasAudio ? "primary" : "disabled", - colorVideo = isValid && media.hasVideo ? "primary" : "disabled"; + const colorAudio = isValid && media?.has_audio ? "primary" : "disabled", + colorVideo = isValid && media?.has_video ? "primary" : "disabled"; if (!peer) { console.log(`media-control - no peer`); @@ -663,17 +684,24 @@ const MediaControl: React.FC = ({ isSelf, peer, className }) } return ( - +
-
+
- {isSelf && ( + {isSelf ? (
{muted && } {!muted && }
- )} - {!isSelf && ( + ) : (
{muted && } {!muted && } @@ -687,6 +715,7 @@ const MediaControl: React.FC = ({ isSelf, peer, className }) {isValid && ( <> = ({ isSelf, peer, className }) e.target.style.transform = `translate(${translate[0]}px, ${translate[1]}px)`; }} /> -