jketreno/ai-voicebot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixing bugs

Browse Source
This commit is contained in:
James Ketr 2025-08-26 17:11:42 -07:00
parent 45fd4c7006
commit b26366eb05
8 changed files with 970 additions and 824 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
client/package.json
View File

@ -17,11 +17,11 @@
"react-moveable": "^0.56.0", "react-moveable": "^0.56.0",
"react-router-dom": "^7.8.2", "react-router-dom": "^7.8.2",
"react-scripts": "5.0.1", "react-scripts": "5.0.1",
"react-use-websocket": "^4.13.0",
"socket.io-client": "^4.8.1", "socket.io-client": "^4.8.1",
"web-vitals": "^5.1.0" "web-vitals": "^5.1.0"
}, },
"devDependencies": { "devDependencies": {
"typescript": "^5.4.5",
"@types/node": "^20.11.30", "@types/node": "^20.11.30",
"@types/react": "^18.2.70", "@types/react": "^18.2.70",
"@types/react-dom": "^18.2.19", "@types/react-dom": "^18.2.19",

243
client/src/App.tsx
View File

@ -1,129 +1,135 @@
import React, { useState, useEffect, KeyboardEvent, useRef } from "react"; import React, { useState, useEffect, KeyboardEvent } from "react";
import { Input, Paper, Typography } from "@mui/material"; import { Input, Paper, Typography } from "@mui/material";
import { GlobalContext, GlobalContextType } from "./GlobalContext"; import { Session } from "./GlobalContext";
import { UserList } from "./UserList"; import { UserList } from "./UserList";
import "./App.css"; import "./App.css";
import { base } from "./Common"; import { ws_base, base } from "./Common";
import { Box, Button } from "@mui/material"; import { Box, Button } from "@mui/material";
import { BrowserRouter as Router, Route, Routes, useParams } from "react-router-dom"; import { BrowserRouter as Router, Route, Routes, useParams } from "react-router-dom";
import useWebSocket, { ReadyState } from "react-use-websocket";
console.log(`AI Voice Chat Build: ${process.env.REACT_APP_AI_VOICECHAT_BUILD}`); console.log(`AI Voice Chat Build: ${process.env.REACT_APP_AI_VOICECHAT_BUILD}`);
type LobbyProps = { type LobbyProps = {
lobbyId: string; session: Session;
sessionId: string;
}; };
const Lobby: React.FC<LobbyProps> = (props: LobbyProps) => {
const { lobbyId, sessionId } = props;
const [editName, setEditName] = useState<string>("");
const [name, setName] = useState<string>("");
const [ws, setWs] = useState<WebSocket | undefined>(undefined);
const [error, setError] = useState<string | null>(null);
const [global, setGlobal] = useState<GlobalContextType>({
connected: false,
ws: undefined,
sessionId: sessionId,
name: "",
chat: [],
});
const onWsMessage = (event: MessageEvent) => { const Lobby: React.FC<LobbyProps> = (props: LobbyProps) => {
const data = JSON.parse(event.data); const { session } = props;
const { lobbyName = "default" } = useParams<{ lobbyName: string }>();
const [lobbyId, setLobbyId] = useState<string | null>(null);
const [editName, setEditName] = useState<string>("");
const [error, setError] = useState<string | null>(null);
const [socketUrl, setSocketUrl] = useState<string | null>(null);
const socket = useWebSocket(socketUrl, {
onOpen: () => console.log("WebSocket connection opened."),
onClose: () => console.log("WebSocket connection closed."),
onError: (event) => console.error("WebSocket error observed:", event),
onMessage: (event) => console.log("WebSocket message received:"),
shouldReconnect: (closeEvent) => true, // Will attempt to reconnect on all close events.
reconnectInterval: 3000,
share: true,
});
const { sendJsonMessage, lastJsonMessage, readyState } = socket;
useEffect(() => {
if (lobbyId && session) {
setSocketUrl(`${ws_base}/${lobbyId}/${session.id}`);
}
}, [lobbyId, session]);
useEffect(() => {
if (!lastJsonMessage) {
return;
}
const data: any = lastJsonMessage;
switch (data.type) { switch (data.type) {
case "update": case "update":
if ("name" in data) { if ("name" in data) {
setName(data.name); console.log(`Lobby - name set to ${data.name}`);
session.name = data.name;
} }
break; break;
case "error": case "error":
console.error(`Lobby - Server error: ${data.error}`);
setError(data.error); setError(data.error);
break; break;
default: default:
break; break;
} }
}; }, [lastJsonMessage]);
const refWsMessage = useRef(onWsMessage);
useEffect(() => { useEffect(() => {
refWsMessage.current = onWsMessage; console.log("WebSocket connection status: ", readyState);
}, [readyState]);
useEffect(() => {
if (!session || !lobbyName) {
return;
}
const getLobbyId = async (lobbyName: string, session: Session) => {
const res = await fetch(`${base}/api/lobby/${lobbyName}/${session.id}`, {
method: "GET",
cache: "no-cache",
credentials: "same-origin",
headers: {
"Content-Type": "application/json",
},
}); });
useEffect(() => { if (res.status >= 400) {
if (!ws) { const error = `Unable to connect to AI Voice Chat server! Try refreshing your browser in a few seconds.`;
return; console.error(error);
setError(error);
} }
const cbMessage = (e: MessageEvent) => refWsMessage.current(e);
ws.addEventListener("message", cbMessage);
return () => {
ws.removeEventListener("message", cbMessage);
};
}, [ws, refWsMessage]);
// Setup websocket connection on mount (only once) const data = await res.json();
useEffect(() => { if (data.error) {
if (!lobbyId) { console.error(`Lobby - Server error: ${data.error}`);
console.log("No lobby ID"); setError(data.error);
return; return;
} }
let loc = window.location,
new_uri;
if (loc.protocol === "https:") {
new_uri = "wss";
} else {
new_uri = "ws";
}
new_uri = `${new_uri}://${loc.host}${base}/ws/lobby/${lobbyId}`;
const socket = new WebSocket(new_uri);
socket.onopen = () => {
console.log("WebSocket connected");
setGlobal((g: GlobalContextType) => ({ ...g, connected: true }));
if (name) {
socket.send(JSON.stringify({ type: "set_name", name }));
}
};
setWs(socket);
setGlobal((g: GlobalContextType) => ({ ...g, ws: socket }));
return () => {
setGlobal((g: GlobalContextType) => ({ ...g, connected: false, ws: undefined }));
if (socket.readyState !== 0) {
socket.close();
}
};
// Only run once on mount
// eslint-disable-next-line
}, []);
// Update global context and send set_name when name changes setLobbyId(data.lobby);
useEffect(() => { };
if (!ws || !global.connected || global.name === name) {
return; getLobbyId(lobbyName, session);
} }, [session, lobbyName, setLobbyId]);
setGlobal((g: GlobalContextType) => ({ ...g, name }));
console.log("Sending set_name", name); const setName = (name: string) => {
ws.send(JSON.stringify({ type: "set_name", name })); sendJsonMessage({
}, [name, ws, global]); type: "set_name",
name: name,
});
};
const handleKeyDown = (event: KeyboardEvent<HTMLTextAreaElement>): void => { const handleKeyDown = (event: KeyboardEvent<HTMLTextAreaElement>): void => {
if (event.key === "Enter") { if (event.key === "Enter") {
event.preventDefault(); event.preventDefault();
if (!editName.trim()) { const newName = editName.trim();
if (!newName || session?.name === newName) {
return; return;
} }
setName(editName.trim()); setName(newName);
setEditName(""); setEditName("");
} }
}; };
return ( return (
<Paper className="Lobby" sx={{ p: 2, m: 2, width: "fit-content" }}> <Paper className="Lobby" sx={{ p: 2, m: 2, width: "fit-content" }}>
{!global.connected ? ( {readyState !== ReadyState.OPEN || !session ? (
<h2>Connecting to server...</h2> <h2>Connecting to server...</h2>
) : ( ) : (
<GlobalContext.Provider value={global}> <>
{global.name && <UserList />} <Box sx={{ mb: 2, display: "flex", gap: 2, alignItems: "flex-start", flexDirection: "column" }}>
<Box>
{!global.name && ( <Typography variant="h5">AI Voice Chat Lobby: {lobbyName}</Typography>
</Box>
<Box>
{session.name && <Typography>You are logged in as: {session.name}</Typography>}
{!session.name && (
<Box sx={{ gap: 1, display: "flex", flexDirection: "column", alignItems: "flex-start" }}> <Box sx={{ gap: 1, display: "flex", flexDirection: "column", alignItems: "flex-start" }}>
<Typography>Enter your name to join:</Typography> <Typography>Enter your name to join:</Typography>
<Box sx={{ display: "flex", gap: 1, width: "100%" }}> <Box sx={{ display: "flex", gap: 1, width: "100%" }}>
@ -139,10 +145,8 @@ const Lobby: React.FC<LobbyProps> = (props: LobbyProps) => {
<Button <Button
variant="contained" variant="contained"
onClick={() => { onClick={() => {
if (ws && global.connected && editName) { setName(editName);
setName(editName.trim());
setEditName(""); setEditName("");
}
}} }}
disabled={!editName.trim()} disabled={!editName.trim()}
> >
@ -151,7 +155,28 @@ const Lobby: React.FC<LobbyProps> = (props: LobbyProps) => {
</Box> </Box>
</Box> </Box>
)} )}
</GlobalContext.Provider> </Box>
</Box>
{session.name && (
<>
{session.lobbies.map((lobby: string) => (
<Box key={lobby}>
<Button
variant="contained"
href={`${base}/${lobby}`}
disabled={lobby === lobbyName}
sx={{ mr: 1, mb: 1 }}
>
{lobby === lobbyName ? `In Lobby: ${lobby}` : `Join Lobby: ${lobby}`}
</Button>
</Box>
))}
{session && socketUrl && <UserList socketUrl={socketUrl} session={session} />}
</>
)}
</>
)} )}
{error && ( {error && (
<Paper className="Error" sx={{ p: 2, m: 2, width: "fit-content", backgroundColor: "#ffdddd" }}> <Paper className="Error" sx={{ p: 2, m: 2, width: "fit-content", backgroundColor: "#ffdddd" }}>
@ -163,48 +188,56 @@ const Lobby: React.FC<LobbyProps> = (props: LobbyProps) => {
}; };
const App = () => { const App = () => {
const [sessionId, setSessionId] = useState(undefined); const [session, setSession] = useState<Session | null>(null);
const [error, setError] = useState<string | null>(null); const [error, setError] = useState<string | null>(null);
const { lobbyId = "default" } = useParams<{ lobbyId: string }>();
useEffect(() => { useEffect(() => {
console.log(`App - sessionId`, sessionId); if (!session) {
}, [sessionId]);
useEffect(() => {
if (sessionId) {
return; return;
} }
fetch(`${base}/api/lobby`, { console.log(`App - sessionId`, session.id);
}, [session]);
useEffect(() => {
if (session) {
return;
}
const getSession = async () => {
const res = await fetch(`${base}/api/session`, {
method: "GET", method: "GET",
cache: "no-cache", cache: "no-cache",
credentials: "same-origin", credentials: "same-origin",
headers: { headers: {
"Content-Type": "application/json", "Content-Type": "application/json",
}, },
}) });
.then((res) => {
if (res.status >= 400) { if (res.status >= 400) {
const error = `Unable to connect to AI Voice Chat server! Try refreshing your browser in a few seconds.`; const error = `Unable to connect to AI Voice Chat server! Try refreshing your browser in a few seconds.`;
console.error(error); console.error(error);
setError(error); setError(error);
return;
} }
return res.json(); const data = await res.json();
}) if (data.error) {
.then((data) => { console.error(`App - Server error: ${data.error}`);
setSessionId(data.session); setError(data.error);
}) return;
.catch((error) => {}); }
}, [sessionId, setSessionId]); setSession(data);
};
getSession();
}, [session, setSession]);
return ( return (
<Box> <Box>
{!sessionId && <h2>Connecting to server...</h2>} {!session && <h2>Connecting to server...</h2>}
{sessionId && ( {session && (
<Router> <Router>
<Routes> <Routes>
<Route element={<Lobby sessionId={sessionId} lobbyId={lobbyId} />} path={`${base}/:lobbyId`} /> <Route element={<Lobby session={session} />} path={`${base}/:lobbyName`} />
<Route element={<Lobby sessionId={sessionId} lobbyId={lobbyId} />} path={`${base}`} /> <Route element={<Lobby session={session} />} path={`${base}`} />
</Routes> </Routes>
</Router> </Router>
)} )}

21
client/src/Common.ts
View File

@ -1,18 +1,5 @@
function debounce<T extends (...args: any[]) => void>(fn: T, ms: number) {
let timer: ReturnType<typeof setTimeout>;
return function(this: any, ...args: Parameters<T>) {
clearTimeout(timer);
timer = setTimeout(() => {
timer = null as any;
fn.apply(this, args);
}, ms);
};
}
const base = process.env.PUBLIC_URL || ""; const base = process.env.PUBLIC_URL || "";
const ws_base: string = `${window.location.protocol === "https:" ? "wss" : "ws"}://${
const assetsPath = `${base}/assets`; window.location.host
const gamesPath = `${base}`; }${base}/ws/lobby`;
export { base, ws_base };
export { base, debounce, assetsPath, gamesPath };
export {};

27
client/src/GlobalContext.tsx
View File

@ -1,21 +1,6 @@
import { createContext } from "react"; type Session = {
id: string;
interface GlobalContextType { name: string | null;
connected: boolean; lobbies: string[];
ws?: WebSocket; };
name?: string; export type { Session };
sessionId?: string;
chat?: any[];
[key: string]: any;
}
const GlobalContext = createContext<GlobalContextType>({
connected: false,
ws: undefined,
name: "",
sessionId: undefined,
chat: []
});
export { GlobalContext };
export type { GlobalContextType };

527
client/src/MediaControl.tsx
View File

@ -1,4 +1,4 @@
import React, { useState, useEffect, useRef, useCallback, useContext } from "react"; import React, { useState, useEffect, useRef, useCallback } from "react";
import Moveable from "react-moveable"; import Moveable from "react-moveable";
import "./MediaControl.css"; import "./MediaControl.css";
import VolumeOff from "@mui/icons-material/VolumeOff"; import VolumeOff from "@mui/icons-material/VolumeOff";
@ -7,27 +7,29 @@ import MicOff from "@mui/icons-material/MicOff";
import Mic from "@mui/icons-material/Mic"; import Mic from "@mui/icons-material/Mic";
import VideocamOff from "@mui/icons-material/VideocamOff"; import VideocamOff from "@mui/icons-material/VideocamOff";
import Videocam from "@mui/icons-material/Videocam"; import Videocam from "@mui/icons-material/Videocam";
import { GlobalContext } from "./GlobalContext";
import Box from "@mui/material/Box"; import Box from "@mui/material/Box";
import useWebSocket from "react-use-websocket";
import { Session } from "./GlobalContext";
const debug = true; const debug = true;
// Types for peer and track context // Types for peer and track context
interface Peer { interface Peer {
sessionId: string; session_id: string;
peerName: string; peerName: string;
hasAudio: boolean; has_audio: boolean;
hasVideo: boolean; has_video: boolean;
attributes: Record<string, any>; attributes: Record<string, any>;
muted: boolean; muted: boolean;
videoOn: boolean; video_on: boolean /* Set by client */;
local: boolean; local: boolean;
dead: boolean; dead: boolean;
connection?: RTCPeerConnection; connection?: RTCPeerConnection;
} }
export type { Peer };
interface TrackContext { interface TrackContext {
media: MediaStream; media: MediaStream | null;
audio: boolean; audio: boolean;
video: boolean; video: boolean;
} }
@ -35,8 +37,8 @@ interface TrackContext {
interface AddPeerConfig { interface AddPeerConfig {
peer_id: string; peer_id: string;
peer_name: string; peer_name: string;
hasAudio: boolean; has_audio: boolean;
hasVideo: boolean; has_video: boolean;
should_create_offer?: boolean; should_create_offer?: boolean;
} }
@ -82,15 +84,48 @@ const Video: React.FC<VideoProps> = ({ srcObject, local, ...props }) => {
}; };
type MediaAgentProps = { type MediaAgentProps = {
socketUrl: string;
session: Session;
peers: Record<string, Peer>;
setPeers: (peers: Record<string, Peer>) => void; setPeers: (peers: Record<string, Peer>) => void;
}; };
const MediaAgent = (props: MediaAgentProps) => { const MediaAgent = (props: MediaAgentProps) => {
const { setPeers } = props; const { peers, setPeers, socketUrl, session } = props;
const { name, ws, sessionId } = useContext(GlobalContext); // track: null = no local media, TrackContext = local media
const [peers] = useState<Record<string, Peer>>({}); const [track, setTrack] = useState<TrackContext | null>(null);
const [track, setTrack] = useState<TrackContext | undefined>(undefined);
const ignore = useRef(false); const { sendJsonMessage, lastJsonMessage } = useWebSocket(socketUrl, {
share: true,
onError: (err) => {
console.error(err);
},
onClose: (_event: CloseEvent) => {
if (!session) {
return;
}
console.log(`media-agent - ${session.name} Disconnected from signaling server`);
/* Tear down all of our peer connections and remove all the
* media divs when we disconnect */
for (let peer_id in peers) {
if (peers[peer_id].local) {
continue;
}
if (peers[peer_id].connection) {
peers[peer_id].connection.close();
peers[peer_id].connection = undefined;
}
}
for (let id in peers) {
peers[id].dead = true;
peers[id].connection = undefined;
}
if (debug) console.log(`media-agent - close`, peers);
setPeers(Object.assign({}, peers));
},
});
const onTrack = useCallback( const onTrack = useCallback(
(event: RTCTrackEvent) => { (event: RTCTrackEvent) => {
@ -102,7 +137,6 @@ const MediaAgent = (props: MediaAgentProps) => {
Object.assign(peers[peer].attributes, { Object.assign(peers[peer].attributes, {
srcObject: event.streams[0] || event.track, srcObject: event.streams[0] || event.track,
}); });
/* Trigger update of MediaControl now that a track is available */
setPeers(Object.assign({}, peers)); setPeers(Object.assign({}, peers));
} }
} }
@ -111,59 +145,32 @@ const MediaAgent = (props: MediaAgentProps) => {
); );
const refOnTrack = useRef(onTrack); const refOnTrack = useRef(onTrack);
const sendMessage = useCallback( const addPeer = useCallback(
(data: LobbyMessage) => { (config: AddPeerConfig) => {
if (!ws) { console.log("media-agent - addPeer - ", { config, peers });
return; if (config.peer_id in peers) {
} if (!peers[config.peer_id].dead) {
ws.send(JSON.stringify(data)); console.log(`media-agent - addPeer - ${config.peer_id} already in peers`);
},
[ws]
);
const onWsMessage = useCallback(
(event: MessageEvent) => {
const addPeer = (config: AddPeerConfig) => {
console.log("media-agent - Signaling server said to add peer:", config);
if (!track) {
console.log(`media-agent - No local media track`);
return;
}
const { peer_id, peer_name } = config;
if (peer_id in peers) {
if (!peers[peer_id].dead) {
/* This is normal when peers are added by other connecting
* peers through the signaling server */
console.log(`media-agent - addPeer - ${peer_id} already in peers`);
return; return;
} }
} }
/* Even if reviving, allocate a new Object so <MediaControl> will
* have its peer state change and trigger an update from
* <PlayerList> */
const peer: Peer = { const peer: Peer = {
sessionId: peer_id, session_id: config.peer_id,
peerName: peer_name, peerName: config.peer_name,
hasAudio: config.hasAudio, has_audio: config.has_audio,
hasVideo: config.hasVideo, has_video: config.has_video,
attributes: {}, attributes: {},
muted: false, muted: false,
videoOn: true, video_on: true,
local: false, local: false,
dead: false, dead: false,
}; };
if (peer_id in peers) { if (config.peer_id in peers) {
peer.muted = peers[peer_id].muted; peer.muted = peers[config.peer_id].muted;
peer.videoOn = peers[peer_id].videoOn; peer.video_on = peers[config.peer_id].video_on;
console.log(`media-agent - addPeer - reviving dead peer ${peer_id}`, peer);
} else {
peer.muted = false;
peer.videoOn = true;
} }
peers[peer_id] = peer; peers[config.peer_id] = peer;
console.log(`media-agent - addPeer - remote`, peers); setPeers({ ...peers });
setPeers(Object.assign({}, peers));
// RTCPeerConnection config should be passed directly, not as 'configuration' property
const connection = new RTCPeerConnection({ const connection = new RTCPeerConnection({
iceServers: [ iceServers: [
{ {
@ -174,21 +181,17 @@ const MediaAgent = (props: MediaAgentProps) => {
], ],
}); });
peer.connection = connection; peer.connection = connection;
connection.addEventListener("connectionstatechange", (event) => { connection.addEventListener("connectionstatechange", (event) => {
console.log(`media-agent - connectionstatechange - `, connection.connectionState, event); console.log(`media-agent - connectionstatechange - `, connection.connectionState, event);
}); });
connection.addEventListener("negotiationneeded", (event) => { connection.addEventListener("negotiationneeded", (event) => {
console.log(`media-agent - negotiationneeded - `, connection.connectionState, event); console.log(`media-agent - negotiationneeded - `, connection.connectionState, event);
}); });
connection.addEventListener("icecandidateerror", (event: RTCPeerConnectionIceErrorEvent) => { connection.addEventListener("icecandidateerror", (event: RTCPeerConnectionIceErrorEvent) => {
if (event.errorCode === 701) { if (event.errorCode === 701) {
if (connection.iceGatheringState === "gathering") { if (connection.iceGatheringState === "gathering") {
console.log(`media-agent - Unable to reach host: ${event.url}`); console.log(`media-agent - Unable to reach host: ${event.url}`);
} else { } else {
// hostcandidate is deprecated and not always present
console.error( console.error(
`media-agent - icecandidateerror - `, `media-agent - icecandidateerror - `,
event.errorCode, event.errorCode,
@ -199,68 +202,50 @@ const MediaAgent = (props: MediaAgentProps) => {
} }
} }
}); });
connection.onicecandidate = (event: RTCPeerConnectionIceEvent) => { connection.onicecandidate = (event: RTCPeerConnectionIceEvent) => {
if (!event.candidate) { if (!event.candidate) {
console.log(`media-agent - icecanditate - gathering is complete: ${connection.connectionState}`); console.log(`media-agent - icecanditate - gathering is complete: ${connection.connectionState}`);
return; return;
} }
// If a srflx candidate was found, notify that the STUN server works! sendJsonMessage({
if (event.candidate && event.candidate.type === "srflx") {
console.log("media-agent - The STUN server is reachable!");
// address is not standard, use candidate.candidate string parsing if needed
}
// If a relay candidate was found, notify that the TURN server works!
if (event.candidate && event.candidate.type === "relay") {
console.log("media-agent - The TURN server is reachable !");
}
console.log(`media-agent - onicecandidate - `, event.candidate);
sendMessage({
type: "relayICECandidate", type: "relayICECandidate",
config: { config: {
peer_id, peer_id: config.peer_id,
candidate: event.candidate, candidate: event.candidate,
}, },
}); });
}; };
connection.ontrack = (e: RTCTrackEvent) => refOnTrack.current(e); connection.ontrack = (e: RTCTrackEvent) => refOnTrack.current(e);
// Only add local tracks if present
// Add all tracks from local media if (track && track.media) {
track.media.getTracks().forEach((t) => { track.media.getTracks().forEach((t) => {
connection.addTrack(t, track.media); connection.addTrack(t, track.media!);
}); });
}
/* Only one side of the peer connection should create the
* offer, the signaling server picks one to be the offerer.
* The other user will get a 'sessionDescription' event and will
* create an offer, then send back an answer 'sessionDescription'
* to us
*/
if (config.should_create_offer) { if (config.should_create_offer) {
if (debug) console.log(`media-agent - Creating RTC offer to ${peer_id}`);
connection connection
.createOffer() .createOffer()
.then((local_description) => { .then((local_description) => {
if (debug) console.log(`media-agent - Local offer description is: `, local_description);
return connection.setLocalDescription(local_description).then(() => { return connection.setLocalDescription(local_description).then(() => {
sendMessage({ sendJsonMessage({
type: "relaySessionDescription", type: "relaySessionDescription",
config: { config: {
peer_id, peer_id: config.peer_id,
session_description: local_description, session_description: local_description,
}, },
}); });
if (debug) console.log(`media-agent - Offer setLocalDescription succeeded`);
}); });
}) })
.catch((error) => { .catch((error) => {
console.error(`media-agent - Offer setLocalDescription failed!`, error); console.error(`media-agent - Offer setLocalDescription failed!`, error);
}); });
} }
}; },
[peers, setPeers, track, sendJsonMessage]
);
const sessionDescription = ({ peer_id, session_description }: SessionDescriptionData) => { const sessionDescription = useCallback(
({ peer_id, session_description }: SessionDescriptionData) => {
const peer = peers[peer_id]; const peer = peers[peer_id];
if (!peer) { if (!peer) {
console.error(`media-agent - sessionDescription - No peer for ${peer_id}`); console.error(`media-agent - sessionDescription - No peer for ${peer_id}`);
@ -281,12 +266,11 @@ const MediaAgent = (props: MediaAgentProps) => {
connection connection
.createAnswer() .createAnswer()
.then((local_description) => { .then((local_description) => {
if (debug) if (debug) console.log(`media-agent - sessionDescription - Answer description is: `, local_description);
console.log(`media-agent - sessionDescription - Answer description is: `, local_description);
connection connection
.setLocalDescription(local_description) .setLocalDescription(local_description)
.then(() => { .then(() => {
sendMessage({ sendJsonMessage({
type: "relaySessionDescription", type: "relaySessionDescription",
config: { config: {
peer_id, peer_id,
@ -307,10 +291,13 @@ const MediaAgent = (props: MediaAgentProps) => {
.catch((error) => { .catch((error) => {
console.log(`media-agent - sessionDescription - setRemoteDescription error: `, error); console.log(`media-agent - sessionDescription - setRemoteDescription error: `, error);
}); });
}; },
[peers, sendJsonMessage]
);
const removePeer = ({ peer_id }: RemovePeerData) => { const removePeer = useCallback(
console.log(`media-agent - removePeer - Signaling server said to ` + `remove peer ${peer_id}`); ({ peer_id }: RemovePeerData) => {
console.log(`media-agent - removePeer - Signaling server said to remove peer ${peer_id}`);
if (peer_id in peers) { if (peer_id in peers) {
if (peers[peer_id].connection) { if (peers[peer_id].connection) {
peers[peer_id].connection.close(); peers[peer_id].connection.close();
@ -323,9 +310,12 @@ const MediaAgent = (props: MediaAgentProps) => {
peers[peer_id].dead = true; peers[peer_id].dead = true;
if (debug) console.log(`media-agent - removePeer`, peers); if (debug) console.log(`media-agent - removePeer`, peers);
setPeers(Object.assign({}, peers)); setPeers(Object.assign({}, peers));
}; },
[peers, setPeers]
);
const iceCandidate = ({ peer_id, candidate }: IceCandidateData) => { const iceCandidate = useCallback(
({ peer_id, candidate }: IceCandidateData) => {
/** /**
* The offerer will send a number of ICE Candidate blobs to the * The offerer will send a number of ICE Candidate blobs to the
* answerer so they can begin trying to find the best path to one * answerer so they can begin trying to find the best path to one
@ -344,9 +334,19 @@ const MediaAgent = (props: MediaAgentProps) => {
.catch((error) => { .catch((error) => {
console.error(error, peer, candidate); console.error(error, peer, candidate);
}); });
}; },
[peers]
);
const data = JSON.parse(event.data); useEffect(() => {
if (!lastJsonMessage) {
return;
}
const data: any = lastJsonMessage;
if (!session) {
return;
}
console.log("media-agent - message", data);
if (["addPeer", "removePeer", "iceCandidate", "sessionDescription"].includes(data.type)) { if (["addPeer", "removePeer", "iceCandidate", "sessionDescription"].includes(data.type)) {
console.log(`media-agent - message - ${data.type}`, peers); console.log(`media-agent - message - ${data.type}`, peers);
} }
@ -366,71 +366,27 @@ const MediaAgent = (props: MediaAgentProps) => {
default: default:
break; break;
} }
}, }, [lastJsonMessage, addPeer, removePeer, iceCandidate, sessionDescription, peers, session]);
[peers, setPeers, track, refOnTrack, sendMessage]
);
const refWsMessage = useRef(onWsMessage);
const onWsClose = (_event: CloseEvent) => {
console.log(`media-agent - ${name} Disconnected from signaling server`);
/* Tear down all of our peer connections and remove all the
* media divs when we disconnect */
for (let peer_id in peers) {
if (peers[peer_id].local) {
continue;
}
if (peers[peer_id].connection) {
peers[peer_id].connection.close();
peers[peer_id].connection = undefined;
}
}
for (let id in peers) {
peers[id].dead = true;
peers[id].connection = undefined;
}
if (debug) console.log(`media-agent - close`, peers);
setPeers(Object.assign({}, peers));
};
const refWsClose = useRef(onWsClose);
useEffect(() => { useEffect(() => {
refWsMessage.current = onWsMessage;
refWsClose.current = onWsClose;
refOnTrack.current = onTrack; refOnTrack.current = onTrack;
}); });
useEffect(() => { useEffect(() => {
if (!ws) { console.log(`media-control - Track changed`, track);
return;
}
console.log(`media-control - Binding to WebSocket`);
const cbMessage = (e: MessageEvent) => refWsMessage.current(e);
ws.addEventListener("message", cbMessage);
const cbClose = (e: CloseEvent) => refWsClose.current(e);
ws.addEventListener("close", cbClose);
return () => {
ws.removeEventListener("message", cbMessage);
ws.removeEventListener("close", cbClose);
};
}, [ws, refWsMessage, refWsClose]);
useEffect(() => {
console.log(`media-control - WebSocket or Track changed`);
const join = () => { const join = () => {
sendMessage({ sendJsonMessage({
type: "join", type: "join",
data: { data: {
has_audio: track?.audio ?? false, has_audio: track && track.audio ? track.audio : false,
has_aideo: track?.video ?? false, has_video: track && track.video ? track.video : false,
}, },
}); });
}; };
if (ws && track) { if (track !== undefined) {
console.log(`media-conterol - issuing join request`); console.log(`media-control - issuing join request: `, track);
for (let peer in peers) { for (let peer in peers) {
if (peers[peer].local && peers[peer].dead) { if (peers[peer].local && peers[peer].dead) {
/* Allocate a new Object so <MediaControl> will trigger */ /* Allocate a new Object so <MediaControl> will trigger */
@ -442,24 +398,24 @@ const MediaAgent = (props: MediaAgentProps) => {
} }
join(); join();
} }
}, [ws, track, peers, setPeers, sendMessage]); }, [track, peers, setPeers, sendJsonMessage]);
useEffect(() => { useEffect(() => {
if (!sessionId) { if (!session) {
return; return;
} }
let update = false; let update = false;
if (track) { if (track) {
if (!(sessionId in peers)) { if (!(session.id in peers)) {
update = true; update = true;
peers[sessionId] = { peers[session.id] = {
peerName: name || "Unknown", peerName: session.name || "Unknown",
sessionId: sessionId, session_id: session.id,
local: true, local: true,
muted: true, muted: true,
videoOn: false, video_on: false,
hasVideo: track.video, has_video: track.video,
hasAudio: track.audio, has_audio: track.audio,
attributes: { attributes: {
local: true, local: true,
srcObject: track.media, srcObject: track.media,
@ -472,7 +428,7 @@ const MediaAgent = (props: MediaAgentProps) => {
/* Renaming the local connection requires the peer to be deleted /* Renaming the local connection requires the peer to be deleted
* and re-established with the signaling server */ * and re-established with the signaling server */
for (let key in peers) { for (let key in peers) {
if (peers[key].local && key !== sessionId) { if (peers[key].local && key !== session.id) {
delete peers[key]; delete peers[key];
update = true; update = true;
} }
@ -482,89 +438,147 @@ const MediaAgent = (props: MediaAgentProps) => {
if (debug) console.log(`media-agent - Setting global peers`, peers); if (debug) console.log(`media-agent - Setting global peers`, peers);
setPeers(Object.assign({}, peers)); setPeers(Object.assign({}, peers));
} }
}, [peers, name, setPeers, track]); }, [peers, setPeers, track, session]);
useEffect(() => { const setup_local_media = async (): Promise<TrackContext> => {
if (!ws || !name) { console.log(`media-agent - Requesting access to local audio / video inputs`);
return; const context: TrackContext = { media: null, audio: true, video: true };
}
type setup_local_media_props = { // Try to get user media with fallback logic
audio?: boolean; while (context.audio || context.video) {
video?: boolean;
}
const setup_local_media = async (
props?: setup_local_media_props
): Promise<TrackContext> => {
const { audio = true, video = true } = props ?? {};
// Ask user for permission to use the computers microphone and/or camera
console.log(
`media-agent - Requesting access to local audio: ${audio} / video: ${video} inputs`
);
try { try {
const media = await navigator.mediaDevices.getUserMedia({ context.media = await navigator.mediaDevices.getUserMedia({
audio, audio: context.audio,
video, video: context.video,
}); });
sendMessage({ type: "media_status", video, audio }); break;
// Optionally apply constraints } catch (error) {
if (video && media.getVideoTracks().length > 0) { console.error(`media-agent - Error accessing local media: `, error);
console.log(`media-agent - Applying video constraints to ${media.getVideoTracks().length} video tracks`); if (context.video) {
media.getVideoTracks().forEach((track) => { console.log(`media-agent - Disabling video and trying again`);
track.applyConstraints({ context.video = false;
} else if (context.audio) {
console.log(`media-agent - Disabling audio and trying again`);
context.audio = false;
} else {
console.log(`media-agent - No media available`);
break;
}
}
}
// Helper function to create a black video track
const createBlackVideoTrack = ({ width = 640, height = 480 } = {}): MediaStreamTrack => {
const canvas = document.createElement("canvas");
canvas.width = width;
canvas.height = height;
const ctx = canvas.getContext("2d");
if (ctx) {
ctx.fillStyle = "black";
ctx.fillRect(0, 0, width, height);
}
const stream = canvas.captureStream(30); // 30 FPS
const track = stream.getVideoTracks()[0];
track.enabled = true;
return track;
};
// Helper function to create a silent audio track
const createSilentAudioTrack = (): MediaStreamTrack => {
const audioContext = new AudioContext();
const oscillator = audioContext.createOscillator();
const gainNode = audioContext.createGain();
const destination = audioContext.createMediaStreamDestination();
// Set gain to 0 for silence
gainNode.gain.value = 0;
// Connect: oscillator -> gain -> destination
oscillator.connect(gainNode);
gainNode.connect(destination);
oscillator.start();
const track = destination.stream.getAudioTracks()[0];
track.enabled = true;
return track;
};
// Process the results and create appropriate media stream
const tracks: MediaStreamTrack[] = [];
let hasRealAudio = false;
let hasRealVideo = false;
if (context.media) {
const audioTracks = context.media.getAudioTracks();
const videoTracks = context.media.getVideoTracks();
if (audioTracks.length > 0) {
tracks.push(audioTracks[0]);
hasRealAudio = true;
console.log("media-agent - Using real audio");
}
if (videoTracks.length > 0) {
// Apply video constraints
videoTracks[0].applyConstraints({
width: { min: 160, max: 320 }, width: { min: 160, max: 320 },
height: { min: 120, max: 240 }, height: { min: 120, max: 240 },
}); });
}); tracks.push(videoTracks[0]);
} hasRealVideo = true;
return { media, audio, video }; console.log("media-agent - Using real video");
} catch (error) {
if (video) {
console.log(`media-agent - Access to audio and video failed. Trying just audio.`);
// Try again with only audio if video failed
return setup_local_media({ audio, video: false });
} else if (audio) {
console.log(`media-agent - Access to audio failed.`);
sendMessage({ type: "media_status", video: false, audio: false });
// Return a dummy context with no media
return { media: new MediaStream(), audio: false, video: false };
} else {
// No media requested or available
sendMessage({ type: "media_status", video: false, audio: false });
return { media: new MediaStream(), audio: false, video: false };
} }
} }
// Add synthetic tracks as needed
if (!hasRealAudio) {
tracks.push(createSilentAudioTrack());
console.log("media-agent - Using synthetic silent audio");
}
if (!hasRealVideo) {
tracks.push(createBlackVideoTrack());
console.log("media-agent - Using synthetic black video");
}
// Create final media stream
context.media = new MediaStream(tracks);
// Update context flags to reflect what we actually have
context.audio = hasRealAudio;
context.video = hasRealVideo;
const mediaType =
hasRealAudio && hasRealVideo
? "audio/video"
: hasRealAudio
? "audio only"
: hasRealVideo
? "video only"
: "synthetic audio/video";
console.log(`media-agent - Final media setup: ${mediaType}`);
return context;
}; };
if (!track) { useEffect(() => {
if (debug) console.log(`media-agent - WebSocket open request. ` + `Attempting to create local media.`); if (!session || !session.name) {
return;
}
if (track === null) {
setup_local_media() setup_local_media()
.then((context) => { .then((context) => {
console.log(`media-agent - local media setup complete`, context); sendJsonMessage({ type: "media_status", ...context, media: undefined });
/* once the user has given us access to their
* microphone/camcorder, join the channel and start peering up */
console.log(`media-agent - ignore set to ${ignore.current}`);
// if (ignore.current) {
// console.log(`media-agent - aborting setting local media`);
// } else {
console.log("media-agent - setTrack called with context:", context);
setTrack(context); setTrack(context);
// }
}) })
.catch((error) => { .catch(() => setTrack(null));
/* user denied access to a/v */
console.error("media-agent - error in setup_local_media", error);
console.log("media-agent - Access denied for audio/video");
});
} }
}, [track, session, sendJsonMessage]);
return () => {
ignore.current = true;
if (!track) {
console.log(`media-agent - abort media setup!`);
}
};
}, [ws, track, name, sendMessage]);
return <></>; return <></>;
}; };
@ -579,13 +593,14 @@ const MediaControl: React.FC<MediaControlProps> = ({ isSelf, peer, className })
const [muted, setMuted] = useState<boolean | undefined>(undefined); const [muted, setMuted] = useState<boolean | undefined>(undefined);
const [videoOn, setVideoOn] = useState<boolean | undefined>(undefined); const [videoOn, setVideoOn] = useState<boolean | undefined>(undefined);
const [target, setTarget] = useState<Element | undefined>(); const [target, setTarget] = useState<Element | undefined>();
const [isValid, setIsValid] = useState<boolean>(false);
const [frame, setFrame] = useState<{ translate: [number, number] }>({ const [frame, setFrame] = useState<{ translate: [number, number] }>({
translate: [0, 0], translate: [0, 0],
}); });
useEffect(() => { useEffect(() => {
if (peer && peer.peerName) { if (peer && peer.peerName) {
const el = document.querySelector(`.MediaControl[data-peer="${peer.sessionId}"]`); const el = document.querySelector(`.MediaControl[data-peer="${peer.session_id}"]`);
setTarget(el ?? undefined); setTarget(el ?? undefined);
} }
}, [setTarget, peer]); }, [setTarget, peer]);
@ -599,12 +614,10 @@ const MediaControl: React.FC<MediaControlProps> = ({ isSelf, peer, className })
return; return;
} }
setMuted(peer.muted); setMuted(peer.muted);
setVideoOn(peer.videoOn); setVideoOn(peer.video_on);
setMedia(peer); setMedia(peer);
}, [peer, setMedia, setMuted, setVideoOn]); }, [peer, setMedia, setMuted, setVideoOn]);
console.log(`media-control - render`);
const toggleMute = (event: React.MouseEvent | React.TouchEvent) => { const toggleMute = (event: React.MouseEvent | React.TouchEvent) => {
if (debug) console.log(`media-control - toggleMute - ${peer.peerName}`, !muted); if (debug) console.log(`media-control - toggleMute - ${peer.peerName}`, !muted);
if (peer) { if (peer) {
@ -617,14 +630,14 @@ const MediaControl: React.FC<MediaControlProps> = ({ isSelf, peer, className })
const toggleVideo = (event: React.MouseEvent | React.TouchEvent) => { const toggleVideo = (event: React.MouseEvent | React.TouchEvent) => {
if (debug) console.log(`media-control - toggleVideo - ${peer.peerName}`, !videoOn); if (debug) console.log(`media-control - toggleVideo - ${peer.peerName}`, !videoOn);
if (peer) { if (peer) {
peer.videoOn = !videoOn; peer.video_on = !videoOn;
if (peer.videoOn && media) { if (peer.video_on && media) {
const video = document.querySelector(`video[data-id="${media.peerName}"]`) as HTMLVideoElement | null; const video = document.querySelector(`video[data-id="${media.peerName}"]`) as HTMLVideoElement | null;
if (video && typeof video.play === "function") { if (video && typeof video.play === "function") {
video.play(); video.play();
} }
} }
setVideoOn(peer.videoOn); setVideoOn(peer.video_on);
} }
event.stopPropagation(); event.stopPropagation();
}; };
@ -636,7 +649,7 @@ const MediaControl: React.FC<MediaControlProps> = ({ isSelf, peer, className })
if (media.attributes.srcObject) { if (media.attributes.srcObject) {
console.log(`media-control - audio enable - ${peer.peerName}:${!muted}`); console.log(`media-control - audio enable - ${peer.peerName}:${!muted}`);
(media.attributes.srcObject.getAudioTracks() as MediaStreamTrack[]).forEach((track: MediaStreamTrack) => { (media.attributes.srcObject.getAudioTracks() as MediaStreamTrack[]).forEach((track: MediaStreamTrack) => {
track.enabled = media.hasAudio && !muted; track.enabled = media.has_audio && !muted;
}); });
} }
}); });
@ -648,14 +661,22 @@ const MediaControl: React.FC<MediaControlProps> = ({ isSelf, peer, className })
if (media.attributes.srcObject) { if (media.attributes.srcObject) {
console.log(`media-control - video enable - ${peer.peerName}:${videoOn}`); console.log(`media-control - video enable - ${peer.peerName}:${videoOn}`);
(media.attributes.srcObject.getVideoTracks() as MediaStreamTrack[]).forEach((track: MediaStreamTrack) => { (media.attributes.srcObject.getVideoTracks() as MediaStreamTrack[]).forEach((track: MediaStreamTrack) => {
track.enabled = Boolean(media.hasVideo) && Boolean(videoOn); track.enabled = Boolean(media.has_video) && Boolean(videoOn);
}); });
} }
}); });
useEffect(() => {
if (!media || !peer || media.dead || !media.attributes || !media.attributes.srcObject) {
setIsValid(false);
console.log(`media-control - no media or peer`);
return;
}
console.log(`media-control - has valid peer`);
setIsValid(true);
}, [media, peer, setIsValid]);
const isValid = media && !media.dead, const colorAudio = isValid && media?.has_audio ? "primary" : "disabled",
colorAudio = isValid && media.hasAudio ? "primary" : "disabled", colorVideo = isValid && media?.has_video ? "primary" : "disabled";
colorVideo = isValid && media.hasVideo ? "primary" : "disabled";
if (!peer) { if (!peer) {
console.log(`media-control - no peer`); console.log(`media-control - no peer`);
@ -663,17 +684,24 @@ const MediaControl: React.FC<MediaControlProps> = ({ isSelf, peer, className })
} }
return ( return (
<Box sx={{ display: 'flex', flexDirection: 'column', alignItems: 'center', border: "3px solid green", minWidth: '200px', minHeight: '100px' }}> <Box
sx={{
display: "flex",
flexDirection: "column",
alignItems: "center",
minWidth: "200px",
minHeight: "100px",
}}
>
<div className={`MediaControlSpacer ${className}`} /> <div className={`MediaControlSpacer ${className}`} />
<div className={`MediaControl ${className}`} data-peer={peer.sessionId}> <div className={`MediaControl ${className}`} data-peer={peer.session_id}>
<div className="Controls"> <div className="Controls">
{isSelf && ( {isSelf ? (
<div onTouchStart={toggleMute} onClick={toggleMute}> <div onTouchStart={toggleMute} onClick={toggleMute}>
{muted && <MicOff color={colorAudio} />} {muted && <MicOff color={colorAudio} />}
{!muted && <Mic color={colorAudio} />} {!muted && <Mic color={colorAudio} />}
</div> </div>
)} ) : (
{!isSelf && (
<div onTouchStart={toggleMute} onClick={toggleMute}> <div onTouchStart={toggleMute} onClick={toggleMute}>
{muted && <VolumeOff color={colorAudio} />} {muted && <VolumeOff color={colorAudio} />}
{!muted && <VolumeUp color={colorAudio} />} {!muted && <VolumeUp color={colorAudio} />}
@ -687,6 +715,7 @@ const MediaControl: React.FC<MediaControlProps> = ({ isSelf, peer, className })
{isValid && ( {isValid && (
<> <>
<Moveable <Moveable
sx={{ border: "3px solid blue" }}
pinchable={true} pinchable={true}
draggable={true} draggable={true}
// Moveable expects HTMLElement or SVGElement, not just Element // Moveable expects HTMLElement or SVGElement, not just Element
@ -722,6 +751,7 @@ const MediaControl: React.FC<MediaControlProps> = ({ isSelf, peer, className })
e.target.style.transform = `translate(${translate[0]}px, ${translate[1]}px)`; e.target.style.transform = `translate(${translate[0]}px, ${translate[1]}px)`;
}} }}
/> />
{media && (
<Video <Video
className="Video" className="Video"
data-id={media.peerName} data-id={media.peerName}
@ -729,6 +759,7 @@ const MediaControl: React.FC<MediaControlProps> = ({ isSelf, peer, className })
srcObject={media.attributes.srcObject} srcObject={media.attributes.srcObject}
{...media.attributes} {...media.attributes}
/> />
)}
</> </>
)} )}
{!isValid && <video className="Video"></video>} {!isValid && <video className="Video"></video>}

46
client/src/UserList.css
View File

@ -1,4 +1,4 @@
.PlayerList { .UserList {
display: flex; display: flex;
position: relative; position: relative;
padding: 0.5em; padding: 0.5em;
@ -7,14 +7,14 @@
margin: 0.25rem 0.25rem 0.25rem 0; margin: 0.25rem 0.25rem 0.25rem 0;
} }
.PlayerList .Name { .UserList .Name {
flex-grow: 1; flex-grow: 1;
text-overflow: ellipsis; text-overflow: ellipsis;
overflow: hidden; overflow: hidden;
white-space: nowrap; white-space: nowrap;
} }
.PlayerList .NoNetwork { .UserList .NoNetwork {
display: flex; display: flex;
justify-self: flex-end; justify-self: flex-end;
width: 1em; width: 1em;
@ -25,29 +25,29 @@
background-repeat: no-repeat; background-repeat: no-repeat;
} }
.PlayerList .Unselected { .UserList .Unselected {
display: flex; display: flex;
flex-direction: column; flex-direction: column;
justify-content: space-around; justify-content: space-around;
align-items: center; align-items: center;
} }
/* Player name in the Unselected list... */ /* User name in the Unselected list... */
.PlayerList .Unselected > div:nth-child(2) > div > div:first-child { .UserList .Unselected > div:nth-child(2) > div > div:first-child {
text-overflow: ellipsis; text-overflow: ellipsis;
white-space: nowrap; white-space: nowrap;
overflow: hidden; overflow: hidden;
max-width: 100%; max-width: 100%;
} }
.PlayerList .Unselected > div:nth-child(2) { .UserList .Unselected > div:nth-child(2) {
display: flex; display: flex;
flex-direction: row; flex-direction: row;
flex-wrap: wrap; flex-wrap: wrap;
justify-content: space-around; justify-content: space-around;
} }
.PlayerList .Unselected > div:nth-child(2) > div { .UserList .Unselected > div:nth-child(2) > div {
justify-content: flex-end; justify-content: flex-end;
display: flex; display: flex;
flex-direction: column; flex-direction: column;
@ -59,36 +59,40 @@
border-radius: 0.25rem; border-radius: 0.25rem;
} }
.PlayerList .Unselected .Self { .UserList .Unselected .Self {
border: 1px solid black; border: 1px solid black;
} }
.PlayerList .PlayerSelector .PlayerColor { .UserList .UserSelector .UserColor {
width: 1em; width: 1em;
height: 1em; height: 1em;
} }
.PlayerList .PlayerSelector { .UserList .UserSelector {
display: inline-flex; display: inline-flex;
flex-wrap: wrap; flex-wrap: wrap;
flex-direction: row; flex-direction: row;
} }
.PlayerList .PlayerSelector.MuiList-padding { .UserList .UserSelector.MuiList-padding {
padding: 0; padding: 0;
} }
.PlayerList .PlayerSelector .MuiTypography-body1 { .UserList .UserSelector .MuiTypography-body1 {
font-size: 0.8rem; font-size: 0.8rem;
/* white-space: nowrap;*/ /* white-space: nowrap;*/
} }
.PlayerList .PlayerSelector .MuiTypography-body2 { .UserList .UserSelector .MuiTypography-body2 {
font-size: 0.7rem; font-size: 0.7rem;
white-space: nowrap; white-space: nowrap;
} }
.PlayerList .PlayerSelector .PlayerEntry { .UserList .UserSelf {
border: 2px solid purple !important;
}
.UserList .UserSelector .UserEntry {
display: flex; display: flex;
flex-direction: column; flex-direction: column;
text-overflow: ellipsis; text-overflow: ellipsis;
@ -103,7 +107,7 @@
justify-content: flex-end; justify-content: flex-end;
} }
.PlayerList .PlayerSelector .PlayerEntry > div:first-child { .UserList .UserSelector .UserEntry > div:first-child {
display: flex; display: flex;
flex-direction: row; flex-direction: row;
align-items: center; align-items: center;
@ -111,27 +115,27 @@
margin-bottom: 0.25em; margin-bottom: 0.25em;
} }
.PlayerList .PlayerEntry[data-selectable=true]:hover { .UserList .UserEntry[data-selectable=true]:hover {
border-color: rgba(0,0,0,0.5); border-color: rgba(0,0,0,0.5);
cursor: pointer; cursor: pointer;
} }
.PlayerList .Players .PlayerToggle { .UserList .Users .UserToggle {
min-width: 5em; min-width: 5em;
display: inline-flex; display: inline-flex;
align-items: flex-end; align-items: flex-end;
flex-direction: column; flex-direction: column;
} }
.PlayerList .PlayerName { .UserList .UserName {
padding: 0.5em; padding: 0.5em;
} }
.PlayerList .Players > * { .UserList .Users > * {
width: 100%; width: 100%;
} }
.PlayerList .Players .nameInput { .UserList .Users .nameInput {
flex-grow: 1; flex-grow: 1;
} }

153
client/src/UserList.tsx
View File

@ -1,77 +1,40 @@
import React, { useState, useEffect, useContext, useRef } from "react"; import React, { useState, useEffect, useCallback } from "react";
import Paper from '@mui/material/Paper'; import Paper from "@mui/material/Paper";
import List from '@mui/material/List'; import List from "@mui/material/List";
import "./UserList.css"; import "./UserList.css";
import { GlobalContext } from "./GlobalContext"; import { MediaControl, MediaAgent, Peer } from "./MediaControl";
import { MediaControl, MediaAgent } from "./MediaControl";
import Box from "@mui/material/Box"; import Box from "@mui/material/Box";
import { Session } from "./GlobalContext";
import useWebSocket from "react-use-websocket";
type User = { type User = {
name: string; name: string;
sessionId: string; session_id: string;
live: boolean; live: boolean;
is_self: boolean /* Client side variable */;
}; };
const UserList: React.FC = () => { type UserListProps = {
const { ws, name, sessionId } = useContext(GlobalContext); socketUrl: string;
const [users, setUsers] = useState<Record<string, User>>({}); session: Session;
const [peers, setPeers] = useState<Record<string, any>>({}); };
useEffect(() => { const UserList: React.FC<UserListProps> = (props: UserListProps) => {
console.log("Peers: ", peers); const { socketUrl, session } = props;
}, [peers]); const [users, setUsers] = useState<User[] | null>(null);
const [peers, setPeers] = useState<Record<string, Peer>>({});
const [videoClass, setVideoClass] = useState<string>("Large");
const onWsMessage = (event: MessageEvent) => { const sortUsers = useCallback(
const data = JSON.parse(event.data); (A: any, B: any) => {
switch (data.type) { if (!session) {
case 'users': return 0;
console.log(`users - lobby update`, data.users);
setUsers(data.users);
break;
default:
break;
} }
};
const refWsMessage = useRef(onWsMessage);
useEffect(() => { refWsMessage.current = onWsMessage; });
useEffect(() => {
if (!ws) {
return;
}
const cbMessage = (e: MessageEvent) => refWsMessage.current(e);
ws.addEventListener('message', cbMessage);
return () => {
ws.removeEventListener('message', cbMessage);
};
}, [ws, refWsMessage]);
useEffect(() => {
if (!ws) {
return;
}
ws.send(JSON.stringify({
type: 'list_users',
}));
}, [ws]);
const userElements: JSX.Element[] = [];
const sortedUsers: any[] = [];
for (let key in users) {
if (users[key]) {
sortedUsers.push(users[key]);
}
}
const sortUsers = (A: any, B: any) => {
/* active user first */ /* active user first */
if (A.name === name) { if (A.name === session.name) {
return -1; return -1;
} }
if (B.name === name) { if (B.name === session.name) {
return +1; return +1;
} }
/* Sort active users first */ /* Sort active users first */
@ -86,37 +49,73 @@ const UserList: React.FC = () => {
return A.color.localeCompare(B.color); return A.color.localeCompare(B.color);
} }
return 0; return 0;
}; },
[session]
);
sortedUsers.sort(sortUsers); // Use the WebSocket hook for lobby events (rely on ws from context, but can use hook for message handling)
const { sendJsonMessage } = useWebSocket(socketUrl, {
share: true,
onMessage: (event) => {
if (!session) {
return;
}
const data = JSON.parse(event.data);
switch (data.type) {
case "users":
console.log(`users - lobby update`, data.users);
const u: User[] = data.users;
u.forEach((user) => {
user.is_self = user.session_id === session.id;
});
u.sort(sortUsers);
setVideoClass(u.length <= 2 ? "Medium" : "Small");
setUsers(u);
break;
default:
break;
}
},
});
const videoClass = sortedUsers.length <= 2 ? 'Medium' : 'Small'; useEffect(() => {
if (users !== null) {
return;
}
sendJsonMessage({
type: "list_users",
});
}, [users, sendJsonMessage]);
sortedUsers.forEach((user : User) => { const userElements: JSX.Element[] = [];
const userName = user.name;
const isSelf = user.sessionId === sessionId; users?.forEach((user: User) => {
console.log(`User: ${userName}, Is Self: ${isSelf}, hasPeer: ${peers[user.sessionId] ? 'Yes' : 'No'}`); console.log(`User: ${user.name}, Is Self: ${user.is_self}, hasPeer: ${peers[user.session_id] ? "Yes" : "No"}`);
userElements.push( userElements.push(
<Box key={userName} sx={{ display: 'flex', flexDirection: 'column', alignItems: 'center', border: "3px solid magenta" }} <Box
className="UserEntry" key={user.name}
sx={{ display: "flex", flexDirection: "column", alignItems: "center", border: "3px solid magenta" }}
className={`UserEntry ${user.is_self ? "UserSelf" : ""}`}
> >
<div> <div>
<div className="Name">{userName ? userName : 'Available' }</div> <div className="Name">{user.name ? user.name : user.session_id}</div>
{ userName && !user.live && <div className="NoNetwork"></div> } {user.name && !user.live && <div className="NoNetwork"></div>}
</div> </div>
{ userName && user.live && peers[user.sessionId] && <MediaControl className={videoClass} peer={peers[user.sessionId]} isSelf={isSelf}/> } {user.name && user.live && peers[user.session_id] ? (
<MediaControl className={videoClass} peer={peers[user.session_id]} isSelf={user.is_self} />
) : (
<video className="Video"></video>
)}
</Box> </Box>
); );
}); });
return ( return (
<Paper className={`UserList ${videoClass}`}> <Paper className={`UserList ${videoClass}`}>
<MediaAgent setPeers={setPeers}/> <MediaAgent {...{ session, socketUrl, peers, setPeers }} />
<List className="UserSelector"> <List className="UserSelector">{userElements}</List>
{ userElements }
</List>
</Paper> </Paper>
); );
} };
export { UserList }; export { UserList };

297
server/main.py
View File

@ -34,19 +34,19 @@ class Session:
self.has_video = False self.has_video = False
def getName(session: Session | None) -> str: def getName(session: Session | None) -> str | None:
if not session: if session and session.name:
return "Admin"
if session.name:
return session.name return session.name
return session.id return None
class Lobby: class Lobby:
def __init__(self, id): def __init__(self, name: str):
self.id = id self.id = secrets.token_hex(16)
self.short = id[:8] self.short = self.id[:8]
self.sessions: dict[str, Session] = {} self.name = name
self.sessions: dict[str, Session] = {} # All lobby members
self.peers: dict[str, Session] = {} # RTC joined peers only
def addSession(self, session: Session): def addSession(self, session: Session):
if session.id not in self.sessions: if session.id not in self.sessions:
@ -72,6 +72,13 @@ def getLobby(lobby_id) -> Lobby | None:
return lobbies.get(lobby_id, None) return lobbies.get(lobby_id, None)
def getLobbyByName(lobby_name) -> Lobby | None:
for lobby in lobbies.values():
if lobby.name == lobby_name:
return lobby
return None
# API endpoints # API endpoints
@app.get(f"{public_url}api/health") @app.get(f"{public_url}api/health")
def health(): def health():
@ -83,31 +90,74 @@ def health():
# A user can be in multiple lobbies, but a session is unique to a single user. # A user can be in multiple lobbies, but a session is unique to a single user.
# A user can change their name, but the session ID remains the same and the name # A user can change their name, but the session ID remains the same and the name
# updates for all lobbies. # updates for all lobbies.
@app.get(f"{public_url}api/lobby") @app.get(f"{public_url}api/session")
async def lobby( async def session(
request: Request, response: Response, session_id: str = Cookie(default=None) request: Request, response: Response, session_id: str = Cookie(default=None)
): ):
if session_id is None: if session_id is None:
session_id = secrets.token_hex(16) session_id = secrets.token_hex(16)
response.set_cookie(key="session_id", value=session_id) response.set_cookie(key="session_id", value=session_id)
# Validate that session_id is a hex string of length 32
elif (
not isinstance(session_id, str)
or len(session_id) != 32
or not all(c in "0123456789abcdef" for c in session_id)
):
return {"error": "Invalid session_id"}
print(f"[{session_id[:8]}]: Browser hand-shake achieved.") print(f"[{session_id[:8]}]: Browser hand-shake achieved.")
if session_id not in sessions: session = getSession(session_id)
sessions[session_id] = Session(session_id) if not session:
logger.info(f"{session_id[:8]}: New session created.") session = Session(session_id)
sessions[session_id] = session
logger.info(f"{getSessionName(session)}: New session created.")
else: else:
name = sessions[session_id].name if sessions[session_id].name else "UNSET" logger.info(f"{getSessionName(session)}: Existing session resumed.")
logger.info(f"{session_id[:8]}: Existing session resumed for {name}.")
return {"session": session_id} return {
"id": session_id,
"name": session.name if session.name else None,
"lobbies": [lobby.name for lobby in sessions[session_id].lobbies.values()],
}
all = "[ all ]" @app.get(public_url + "api/lobby/{lobby_name}/{session_id}")
info = "[ info ]" async def lobby(
todo = "[ todo ]" request: Request,
response: Response,
lobby_name: str | None = Path(...),
session_id: str | None = Path(...),
):
if lobby_name is None:
return {"error": "Missing lobby_name"}
if session_id is None:
return {"error": "Missing session_id"}
session = getSession(session_id)
if not session:
return {"error": f"Session not found ({session_id})"}
lobby = getLobbyByName(lobby_name)
if not lobby:
lobby = Lobby(lobby_name)
lobbies[lobby.id] = lobby
logger.info(
f"{getSessionName(session)} <- lobby_create({lobby.short}:{lobby.name})"
)
lobby.addSession(sessions[session_id])
sessions[session_id].lobbies[lobby.id] = lobby
return {"lobby": lobby.id}
all_label = "[ all ]"
info_label = "[ info ]"
todo_label = "[ todo ]"
unset_label = "[ ---- ]"
# Join the media session in a lobby
async def join( async def join(
lobby: Lobby, lobby: Lobby,
session: Session, session: Session,
@ -116,33 +166,34 @@ async def join(
): ):
if not session.name: if not session.name:
logger.error( logger.error(
f"{session.short}:[UNSET] <- join - No name set yet. Audio not available." f"{session.short}:[UNSET] <- join - No name set yet. Media not available."
) )
return return
if not session.ws: if not session.ws:
logger.error( logger.error(
f"{session.short}:{session.name} - No WebSocket connection. Audio not available." f"{session.short}:{session.name} - No WebSocket connection. Media not available."
) )
return return
logger.info(f"{lobby.short}: <- join - {session.short}:{session.name}") logger.info(f"{getSessionName(session)} <- join({getLobbyName(lobby)})")
if session.id in lobby.sessions: if session.id in lobby.peers:
logger.info(f"{session.short}:{session.name} - Already joined to Audio.") logger.info(f"{getSessionName(session)} - Already joined to Media.")
return return
for peer in lobby.sessions.values(): # Notify all existing RTC peers
if not peer.ws: for peer_session in lobby.peers.values():
if not peer_session.ws:
logger.warning( logger.warning(
f"{peer.short}:{peer.name} - No WebSocket connection. Skipping." f"{getSessionName(peer_session)} - No WebSocket connection. Skipping."
) )
continue continue
logger.info( logger.info(
f"{lobby.short}:{peer.name} -> addPeer - {session.short}:{session.name}" f"{getSessionName(peer_session)} -> addPeer({getSessionName(session), getLobbyName(lobby)})"
) )
# Add this caller to all peers await peer_session.ws.send_json(
await peer.ws.send_json(
{ {
"type": "addPeer", "type": "addPeer",
"data": { "data": {
@ -156,100 +207,154 @@ async def join(
) )
# Add each other peer to the caller # Add each other peer to the caller
if session.ws:
logger.info(
f"{getSessionName(session)} -> addPeer({getSessionName(peer_session), getLobbyName(lobby)})"
)
await session.ws.send_json( await session.ws.send_json(
{ {
"type": "addPeer", "type": "addPeer",
"data": { "data": {
"peer_id": peer.id, "peer_id": peer_session.id,
"peer_name": peer.name, "peer_name": peer_session.name,
"should_create_offer": True, "should_create_offer": True,
"has_audio": peer.has_audio, "has_audio": peer_session.has_audio,
"has_video": peer.has_video, "has_video": peer_session.has_video,
}, },
} }
) )
# Add this user as a peer connected to this WebSocket # Add this user as an RTC peer
lobby.sessions[session.id] = session lobby.peers[session.id] = session
await update_users(lobby)
async def part( async def part(
lobby: Lobby, lobby: Lobby,
session: Session, session: Session,
): ):
if not session.ws: if session.id not in lobby.peers:
logger.error( logger.info(
f"{session.id}:{session.name} - No WebSocket connection. Audio not available." f"{getSessionName(session)}: <- part({getLobbyName(lobby)}) - Does not exist in RTC peers."
) )
return return
if session.id not in lobby.sessions: logger.info(
logger.info(f"{session.id}: <- {session.name} - Does not exist in lobby audio.") f"{getSessionName(session)}: <- part({getLobbyName(lobby)}) - Media part."
return )
logger.info(f"{session.id}: <- {session.name} - Audio part.") del lobby.peers[session.id]
logger.info(f"{lobby.short}: -> remove_peer - {session.short}:{session.name}")
del lobby.sessions[session.id] # Remove this peer from all other RTC peers, and remove each peer from this peer
for peer_session in lobby.peers.values():
# Remove this peer from all other peers, and remove each peer from this peer if not peer_session.ws:
for peer in lobby.sessions.values():
if not peer.ws:
logger.warning( logger.warning(
f"{peer.short}:{peer.name} - No WebSocket connection. Skipping." f"{getSessionName(peer_session)} <- part({getLobbyName(lobby)}) - No WebSocket connection. Skipping."
) )
continue continue
await peer.ws.send_json( logger.info(
f"{getSessionName(peer_session)} <- remove_peer({getSessionName(session)})"
)
await peer_session.ws.send_json(
{"type": "remove_peer", "data": {"peer_id": session.id}} {"type": "remove_peer", "data": {"peer_id": session.id}}
) )
if session.ws: if session.ws:
logger.info(
f"{getSessionName(session)} <- remove_peer({getSessionName(peer_session)})"
)
await session.ws.send_json( await session.ws.send_json(
{"type": "remove_peer", "data": {"peer_id": peer.id}} {"type": "remove_peer", "data": {"peer_id": peer_session.id}}
) )
else: else:
logger.warning(f"{session.short}:{session.name} - No WebSocket connection.") logger.error(
f"{getSessionName(session)} <- part({getLobbyName(lobby)}) - No WebSocket connection."
)
async def update_users(lobby: Lobby, requesting_session: Session | None = None):
users = [
{"name": s.name, "live": True if s.ws else False, "session_id": s.id}
for s in lobby.sessions.values()
if s.name
]
if requesting_session:
logger.info(
f"{requesting_session.short}:{requesting_session.name} -> list_users({lobby.name})"
)
if requesting_session.ws:
await requesting_session.ws.send_json({"type": "users", "users": users})
else:
logger.warning(
f"{requesting_session.short}:{requesting_session.name} - No WebSocket connection."
)
else:
for s in lobby.sessions.values():
logger.info(
f"{s.short}:{s.name if s.name else unset_label} -> list_users({lobby.name})"
)
if s.ws:
await s.ws.send_json({"type": "users", "users": users})
def getSessionName(session: Session) -> str:
return f"{session.short}:{session.name if session.name else unset_label}"
def getLobbyName(lobby: Lobby) -> str:
return f"{lobby.short}:{lobby.name}"
# Register websocket endpoint directly on app with full public_url path # Register websocket endpoint directly on app with full public_url path
@app.websocket(f"{public_url}" + "ws/lobby/{lobby_id}") @app.websocket(f"{public_url}" + "ws/lobby/{lobby_id}/{session_id}")
async def websocket_lobby( async def websocket_lobby(
websocket: WebSocket, websocket: WebSocket,
lobby_id: str | None = Path(...), lobby_id: str | None = Path(...),
session_id: str = Cookie(default=None), session_id: str | None = Path(...),
): ):
await websocket.accept() await websocket.accept()
if session_id is None: if lobby_id is None:
await websocket.send_json( await websocket.send_json(
{"type": "error", "error": "Invalid or missing user session"} {"type": "error", "error": "Invalid or missing lobby"}
)
await websocket.close()
return
if session_id is None:
await websocket.send_json(
{"type": "error", "error": "Invalid or missing session"}
) )
await websocket.close() await websocket.close()
return return
short = session_id[:8]
logger.info(f"Session ID from cookie: {session_id}")
session = getSession(session_id) session = getSession(session_id)
if not session: if not session:
logger.error(f"{short}: Invalid session ID {session_id}") logger.error(f"Invalid session ID {session_id}")
await websocket.send_json( await websocket.send_json(
{"type": "error", "error": f"Invalid session ID {session_id}"} {"type": "error", "error": f"Invalid session ID {session_id}"}
) )
await websocket.close() await websocket.close()
return return
session.ws = websocket
if lobby_id is None:
lobby_id = "default"
lobby = getLobby(lobby_id) lobby = getLobby(lobby_id)
if not lobby: if not lobby:
lobby = Lobby(lobby_id) logger.error(f"Invalid lobby ID {lobby_id}")
lobbies[lobby_id] = lobby await websocket.send_json(
logger.info(f"{short}: Lobby {lobby_id} - New Lobby") {"type": "error", "error": f"Invalid lobby ID {lobby_id}"}
)
await websocket.close()
return
logger.info(
f"{getSessionName(session)} <- lobby_connect({lobby.short}:{lobby.name})"
)
session.ws = websocket
# This user session just went from Dead to Live, so update everyone's user list
await update_users(lobby)
try: try:
while True: while True:
data = await websocket.receive_json() data = await websocket.receive_json()
# logger.info(f"{getSessionName(session)} <- RAW Rx: {data}")
match data.get("type"): match data.get("type"):
case "set_name": case "set_name":
name = data.get("name") name = data.get("name")
@ -266,20 +371,20 @@ async def websocket_lobby(
continue continue
session.name = name session.name = name
logger.info(f"{session.short}: Name set to {session.name}") logger.info(
f"{getSessionName(session)} <- set_name({session.name})"
)
await websocket.send_json({"type": "update", "name": name}) await websocket.send_json({"type": "update", "name": name})
await update_users(lobby)
case "list_users": case "list_users":
users = [ await update_users(lobby, session)
{"name": s.name, "live": True, "sessionId": s.id}
for s in sessions.values()
]
await websocket.send_json({"type": "users", "users": users})
case "media_status": case "media_status":
has_audio = data.get("audio", False) has_audio = data.get("audio", False)
has_video = data.get("video", False) has_video = data.get("video", False)
logger.info( logger.info(
f"{session.short}: <- media-status - audio: {has_audio}, video: {has_video}" f"{getSessionName(session)}: <- media-status(audio: {has_audio}, video: {has_video})"
) )
session.has_audio = has_audio session.has_audio = has_audio
session.has_video = has_video session.has_video = has_video
@ -293,10 +398,10 @@ async def websocket_lobby(
await part(lobby, session) await part(lobby, session)
case "relayICECandidate": case "relayICECandidate":
logger.info(data) logger.info(f"{getSessionName(session)} <- relayICECandidate")
if session.id not in lobby.sessions: if session.id not in lobby.peers:
logger.error( logger.error(
f"{session.short}:{session.name} <- relayICECandidate - Does not have Audio" f"{session.short}:{session.name} <- relayICECandidate - Not an RTC peer"
) )
return return
@ -308,21 +413,22 @@ async def websocket_lobby(
"data": {"peer_id": session.id, "candidate": candidate}, "data": {"peer_id": session.id, "candidate": candidate},
} }
if peer_id in lobby.sessions: if peer_id in lobby.peers:
ws = lobby.sessions[peer_id].ws ws = lobby.peers[peer_id].ws
if not ws: if not ws:
logger.warning( logger.warning(
f"{lobby.sessions[peer_id].short}:{lobby.sessions[peer_id].name} - No WebSocket connection. Skipping." f"{lobby.peers[peer_id].short}:{lobby.peers[peer_id].name} - No WebSocket connection. Skipping."
) )
continue return
await ws.send_json(message) await ws.send_json(message)
case "relaySessionDescription": case "relaySessionDescription":
# todo: if audio doesn't work, figure out if its because of peer_id/session_description missing logger.info(f"{getSessionName(session)} <- relaySessionDescription")
if session.id not in lobby.sessions: if session.id not in lobby.peers:
logger.error( logger.error(
f"{session.short}:{session.name} - relaySessionDescription - Does not have Audio" f"{session.short}:{session.name} - relaySessionDescription - Not an RTC peer"
) )
return
peer_id = data.get("config", {}).get("peer_id") peer_id = data.get("config", {}).get("peer_id")
session_description = data.get("config", {}).get( session_description = data.get("config", {}).get(
"session_description" "session_description"
@ -334,13 +440,13 @@ async def websocket_lobby(
"session_description": session_description, "session_description": session_description,
}, },
} }
if peer_id in lobby.sessions: if peer_id in lobby.peers:
ws = lobby.sessions[peer_id].ws ws = lobby.peers[peer_id].ws
if not ws: if not ws:
logger.warning( logger.warning(
f"{lobby.sessions[peer_id].short}:{lobby.sessions[peer_id].name} - No WebSocket connection. Skipping." f"{lobby.peers[peer_id].short}:{lobby.peers[peer_id].name} - No WebSocket connection. Skipping."
) )
continue return
await ws.send_json(message) await ws.send_json(message)
case _: case _:
@ -352,11 +458,12 @@ async def websocket_lobby(
) )
except WebSocketDisconnect: except WebSocketDisconnect:
logger.info(f"WebSocket disconnected for user {session_id}") logger.info(f"{getSessionName(session)} <- WebSocket disconnected for user.")
# Cleanup: remove session from lobby and sessions dict # Cleanup: remove session from lobby and sessions dict
session.ws = None session.ws = None
if lobby and session: if lobby and session:
await part(lobby, session) await part(lobby, session)
await update_users(lobby)
# if session_id in sessions: # if session_id in sessions:
# del sessions[session_id] # del sessions[session_id]
@ -421,7 +528,7 @@ else:
@app.websocket("/ws") @app.websocket("/ws")
async def websocket_proxy(websocket: StarletteWebSocket): async def websocket_proxy(websocket: StarletteWebSocket):
logger.info("WebSocket proxy connection established.") logger.info("REACT: WebSocket proxy connection established.")
# Get scheme from websocket.url (should be 'ws' or 'wss') # Get scheme from websocket.url (should be 'ws' or 'wss')
scheme = websocket.url.scheme if hasattr(websocket, "url") else "ws" scheme = websocket.url.scheme if hasattr(websocket, "url") else "ws"
target_url = f"{scheme}://static-frontend:3000/ws" target_url = f"{scheme}://static-frontend:3000/ws"
@ -449,7 +556,7 @@ else:
try: try:
await asyncio.gather(client_to_server(), server_to_client()) await asyncio.gather(client_to_server(), server_to_client())
except (WebSocketDisconnect, websockets.ConnectionClosed): except (WebSocketDisconnect, websockets.ConnectionClosed):
logger.info("WebSocket proxy connection closed.") logger.info("REACT: WebSocket proxy connection closed.")
except Exception as e: except Exception as e:
logger.error(f"WebSocket proxy error: {e}") logger.error(f"REACT: WebSocket proxy error: {e}")
await websocket.close() await websocket.close()