jketreno/ai-voicebot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Audio Video almost working; need to merge "users" and "peers"

Browse Source
This commit is contained in:
James Ketr 2025-08-24 15:38:49 -07:00
parent 642935764c
commit 45fd4c7006
5 changed files with 106 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
client/src/App.tsx
View File

@ -23,17 +23,19 @@ const Lobby: React.FC<LobbyProps> = (props: LobbyProps) => {
const [global, setGlobal] = useState<GlobalContextType>({
connected: false,
ws: undefined,
sessionId: sessionId,
name: "",
chat: [],
});
useEffect(() => {
console.log(global);
}, [global]);
const onWsMessage = (event: MessageEvent) => {
const data = JSON.parse(event.data);
switch (data.type) {
case "update":
if ("name" in data) {
setName(data.name);
}
break;
case "error":
setError(data.error);
break;

2
client/src/GlobalContext.tsx
View File

@ -4,6 +4,7 @@ interface GlobalContextType {
connected: boolean;
ws?: WebSocket;
name?: string;
sessionId?: string;
chat?: any[];
[key: string]: any;
}
@ -12,6 +13,7 @@ const GlobalContext = createContext<GlobalContextType>({
connected: false,
ws: undefined,
name: "",
sessionId: undefined,
chat: []
});

78
client/src/MediaControl.tsx
View File

@ -14,7 +14,8 @@ const debug = true;
// Types for peer and track context
interface Peer {
name: string;
sessionId: string;
peerName: string;
hasAudio: boolean;
hasVideo: boolean;
attributes: Record<string, any>;
@ -33,6 +34,7 @@ interface TrackContext {
interface AddPeerConfig {
peer_id: string;
peer_name: string;
hasAudio: boolean;
hasVideo: boolean;
should_create_offer?: boolean;
@ -85,7 +87,7 @@ type MediaAgentProps = {
const MediaAgent = (props: MediaAgentProps) => {
const { setPeers } = props;
const { name, ws } = useContext(GlobalContext);
const { name, ws, sessionId } = useContext(GlobalContext);
const [peers] = useState<Record<string, Peer>>({});
const [track, setTrack] = useState<TrackContext | undefined>(undefined);
const ignore = useRef(false);
@ -127,7 +129,7 @@ const MediaAgent = (props: MediaAgentProps) => {
console.log(`media-agent - No local media track`);
return;
}
const peer_id = config.peer_id;
const { peer_id, peer_name } = config;
if (peer_id in peers) {
if (!peers[peer_id].dead) {
/* This is normal when peers are added by other connecting
@ -140,7 +142,8 @@ const MediaAgent = (props: MediaAgentProps) => {
* have its peer state change and trigger an update from
* <PlayerList> */
const peer: Peer = {
name: peer_id,
sessionId: peer_id,
peerName: peer_name,
hasAudio: config.hasAudio,
hasVideo: config.hasVideo,
attributes: {},
@ -442,15 +445,16 @@ const MediaAgent = (props: MediaAgentProps) => {
}, [ws, track, peers, setPeers, sendMessage]);
useEffect(() => {
if (!name) {
if (!sessionId) {
return;
}
let update = false;
if (track) {
if (!(name in peers)) {
if (!(sessionId in peers)) {
update = true;
peers[name] = {
name: name,
peers[sessionId] = {
peerName: name || "Unknown",
sessionId: sessionId,
local: true,
muted: true,
videoOn: false,
@ -468,7 +472,7 @@ const MediaAgent = (props: MediaAgentProps) => {
/* Renaming the local connection requires the peer to be deleted
* and re-established with the signaling server */
for (let key in peers) {
if (peers[key].local && key !== name) {
if (peers[key].local && key !== sessionId) {
delete peers[key];
update = true;
}
@ -485,21 +489,27 @@ const MediaAgent = (props: MediaAgentProps) => {
return;
}
type setup_local_media_props = {
audio?: boolean;
video?: boolean;
}
const setup_local_media = async (
options: { audio?: boolean; video?: boolean } = { audio: true, video: true }
props?: setup_local_media_props
): Promise<TrackContext> => {
const { audio = true, video = true } = props ?? {};
// Ask user for permission to use the computers microphone and/or camera
console.log(
`media-agent - Requesting access to local audio: ${!!options.audio} / video: ${!!options.video} inputs`
`media-agent - Requesting access to local audio: ${audio} / video: ${video} inputs`
);
try {
const media = await navigator.mediaDevices.getUserMedia({
audio: !!options.audio,
video: !!options.video,
audio,
video,
});
sendMessage({ type: "media_status", video: !!options.video, audio: !!options.audio });
sendMessage({ type: "media_status", video, audio });
// Optionally apply constraints
if (options.video && media.getVideoTracks().length > 0) {
if (video && media.getVideoTracks().length > 0) {
console.log(`media-agent - Applying video constraints to ${media.getVideoTracks().length} video tracks`);
media.getVideoTracks().forEach((track) => {
track.applyConstraints({
width: { min: 160, max: 320 },
@ -507,13 +517,13 @@ const MediaAgent = (props: MediaAgentProps) => {
});
});
}
return { media, audio: !!options.audio, video: !!options.video };
return { media, audio, video };
} catch (error) {
if (options.video) {
if (video) {
console.log(`media-agent - Access to audio and video failed. Trying just audio.`);
// Try again with only audio if video failed
return setup_local_media({ audio: options.audio, video: false });
} else if (options.audio) {
return setup_local_media({ audio, video: false });
} else if (audio) {
console.log(`media-agent - Access to audio failed.`);
sendMessage({ type: "media_status", video: false, audio: false });
// Return a dummy context with no media
@ -530,14 +540,16 @@ const MediaAgent = (props: MediaAgentProps) => {
if (debug) console.log(`media-agent - WebSocket open request. ` + `Attempting to create local media.`);
setup_local_media()
.then((context) => {
console.log(`media-agent - local media setup complete`, context);
/* once the user has given us access to their
* microphone/camcorder, join the channel and start peering up */
if (ignore.current) {
console.log(`media-agent - aborting setting local media`);
} else {
console.log(`media-agent - ignore set to ${ignore.current}`);
// if (ignore.current) {
// console.log(`media-agent - aborting setting local media`);
// } else {
console.log("media-agent - setTrack called with context:", context);
setTrack(context);
}
// }
})
.catch((error) => {
/* user denied access to a/v */
@ -572,8 +584,8 @@ const MediaControl: React.FC<MediaControlProps> = ({ isSelf, peer, className })
});
useEffect(() => {
if (peer && peer.name) {
const el = document.querySelector(`.MediaControl[data-peer="${peer.name}"]`);
if (peer && peer.peerName) {
const el = document.querySelector(`.MediaControl[data-peer="${peer.sessionId}"]`);
setTarget(el ?? undefined);
}
}, [setTarget, peer]);
@ -594,7 +606,7 @@ const MediaControl: React.FC<MediaControlProps> = ({ isSelf, peer, className })
console.log(`media-control - render`);
const toggleMute = (event: React.MouseEvent | React.TouchEvent) => {
if (debug) console.log(`media-control - toggleMute - ${peer.name}`, !muted);
if (debug) console.log(`media-control - toggleMute - ${peer.peerName}`, !muted);
if (peer) {
peer.muted = !muted;
setMuted(peer.muted);
@ -603,11 +615,11 @@ const MediaControl: React.FC<MediaControlProps> = ({ isSelf, peer, className })
};
const toggleVideo = (event: React.MouseEvent | React.TouchEvent) => {
if (debug) console.log(`media-control - toggleVideo - ${peer.name}`, !videoOn);
if (debug) console.log(`media-control - toggleVideo - ${peer.peerName}`, !videoOn);
if (peer) {
peer.videoOn = !videoOn;
if (peer.videoOn && media) {
const video = document.querySelector(`video[data-id="${media.name}"]`) as HTMLVideoElement | null;
const video = document.querySelector(`video[data-id="${media.peerName}"]`) as HTMLVideoElement | null;
if (video && typeof video.play === "function") {
video.play();
}
@ -622,7 +634,7 @@ const MediaControl: React.FC<MediaControlProps> = ({ isSelf, peer, className })
return;
}
if (media.attributes.srcObject) {
console.log(`media-control - audio enable - ${peer.name}:${!muted}`);
console.log(`media-control - audio enable - ${peer.peerName}:${!muted}`);
(media.attributes.srcObject.getAudioTracks() as MediaStreamTrack[]).forEach((track: MediaStreamTrack) => {
track.enabled = media.hasAudio && !muted;
});
@ -634,7 +646,7 @@ const MediaControl: React.FC<MediaControlProps> = ({ isSelf, peer, className })
return;
}
if (media.attributes.srcObject) {
console.log(`media-control - video enable - ${peer.name}:${videoOn}`);
console.log(`media-control - video enable - ${peer.peerName}:${videoOn}`);
(media.attributes.srcObject.getVideoTracks() as MediaStreamTrack[]).forEach((track: MediaStreamTrack) => {
track.enabled = Boolean(media.hasVideo) && Boolean(videoOn);
});
@ -651,9 +663,9 @@ const MediaControl: React.FC<MediaControlProps> = ({ isSelf, peer, className })
}
return (
<Box sx={{ display: 'flex', flexDirection: 'column', alignItems: 'center', border: "3px solid green" }}>
<Box sx={{ display: 'flex', flexDirection: 'column', alignItems: 'center', border: "3px solid green", minWidth: '200px', minHeight: '100px' }}>
<div className={`MediaControlSpacer ${className}`} />
<div className={`MediaControl ${className}`} data-peer={peer.name}>
<div className={`MediaControl ${className}`} data-peer={peer.sessionId}>
<div className="Controls">
{isSelf && (
<div onTouchStart={toggleMute} onClick={toggleMute}>
@ -712,7 +724,7 @@ const MediaControl: React.FC<MediaControlProps> = ({ isSelf, peer, className })
/>
<Video
className="Video"
data-id={media.name}
data-id={media.peerName}
autoPlay={true}
srcObject={media.attributes.srcObject}
{...media.attributes}

54
client/src/UserList.tsx
View File

@ -6,12 +6,15 @@ import { GlobalContext } from "./GlobalContext";
import { MediaControl, MediaAgent } from "./MediaControl";
import Box from "@mui/material/Box";
type User = {
name: string;
sessionId: string;
live: boolean;
};
const UserList: React.FC = () => {
const { ws, name } = useContext(GlobalContext);
const [users, setUsers] = useState<Record<string, any>>({});
const [unselected, setUneslected] = useState<string[]>([]);
const [state, setState] = useState<string>('lobby');
const [color, setColor] = useState<string | undefined>(undefined);
const { ws, name, sessionId } = useContext(GlobalContext);
const [users, setUsers] = useState<Record<string, User>>({});
const [peers, setPeers] = useState<Record<string, any>>({});
useEffect(() => {
@ -55,7 +58,6 @@ const UserList: React.FC = () => {
const userElements: JSX.Element[] = [];
const inLobby = state === 'lobby';
const sortedUsers: any[] = [];
for (let key in users) {
@ -88,59 +90,31 @@ const UserList: React.FC = () => {
sortedUsers.sort(sortUsers);
/* Array of just names... */
unselected.sort((A: string, B: string) => {
/* active user first */
if (A === name) {
return -1;
}
if (B === name) {
return +1;
}
/* Then sort alphabetically */
return A.localeCompare(B);
});
const videoClass = sortedUsers.length <= 2 ? 'Medium' : 'Small';
sortedUsers.forEach(user => {
sortedUsers.forEach((user : User) => {
const userName = user.name;
const selectable = inLobby && (user.status === 'Not active' || color === user.color);
const isSelf = user.sessionId === sessionId;
console.log(`User: ${userName}, Is Self: ${isSelf}, hasPeer: ${peers[user.sessionId] ? 'Yes' : 'No'}`);
userElements.push(
<Box sx={{ display: 'flex', flexDirection: 'column', alignItems: 'center', border: "3px solid magenta" }}
data-selectable={selectable}
data-selected={user.color === color}
<Box key={userName} sx={{ display: 'flex', flexDirection: 'column', alignItems: 'center', border: "3px solid magenta" }}
className="UserEntry"
key={`user-${user.color}`}>
>
<div>
<div className="Name">{userName ? userName : 'Available' }</div>
{ userName && !user.live && <div className="NoNetwork"></div> }
</div>
{ userName && user.live && peers[userName] && <MediaControl className={videoClass} peer={peers[userName]} isSelf={user.color === color}/> }
{ !userName && <div></div> }
{ userName && user.live && peers[user.sessionId] && <MediaControl className={videoClass} peer={peers[user.sessionId]} isSelf={isSelf}/> }
</Box>
);
});
const waiting = unselected.map((user) => {
return <div className={user === name ? 'Self' : ''} key={user}>
<div>{ user }</div>
{peers[user] && <MediaControl className={'Small'} peer={peers[user]} isSelf={name === user}/>}
</div>
});
return (
<Paper className={`UserList ${videoClass}`}>
<MediaAgent setPeers={setPeers}/>
<List className="UserSelector">
{ userElements }
</List>
{ unselected && unselected.length !== 0 && <div className="Unselected">
<div>In lobby</div>
<div>
{ waiting }
</div>
</div> }
</Paper>
);
}

60
server/main.py
View File

@ -132,20 +132,22 @@ async def join(
logger.info(f"{session.short}:{session.name} - Already joined to Audio.")
return
logger.info(f"{lobby.short}: -> addPeer - {session.short}:{session.name}")
for peer in lobby.sessions.values():
if not peer.ws:
logger.warning(
f"{peer.short}:{peer.name} - No WebSocket connection. Skipping."
)
continue
logger.info(
f"{lobby.short}:{peer.name} -> addPeer - {session.short}:{session.name}"
)
# Add this caller to all peers
await peer.ws.send_json(
{
"type": "addPeer",
"data": {
"peer_id": session.id,
"peer_name": session.name,
"should_create_offer": False,
"has_audio": has_audio,
"has_video": has_video,
@ -158,7 +160,8 @@ async def join(
{
"type": "addPeer",
"data": {
"peer_id": peer,
"peer_id": peer.id,
"peer_name": peer.name,
"should_create_offer": True,
"has_audio": peer.has_audio,
"has_video": peer.has_video,
@ -228,7 +231,9 @@ async def websocket_lobby(
session = getSession(session_id)
if not session:
logger.error(f"{short}: Invalid session ID {session_id}")
await websocket.send_json({"type": "error", "error": f"Invalid session ID {session_id}"})
await websocket.send_json(
{"type": "error", "error": f"Invalid session ID {session_id}"}
)
await websocket.close()
return
session.ws = websocket
@ -264,13 +269,18 @@ async def websocket_lobby(
logger.info(f"{session.short}: Name set to {session.name}")
await websocket.send_json({"type": "update", "name": name})
case "list_users":
users = [{"name": s.name, "live": True} for s in sessions.values()]
users = [
{"name": s.name, "live": True, "sessionId": s.id}
for s in sessions.values()
]
await websocket.send_json({"type": "users", "users": users})
case 'media_status':
case "media_status":
has_audio = data.get("audio", False)
has_video = data.get("video", False)
logger.info(f"{session.short}: <- media-status - audio: {has_audio}, video: {has_video}")
logger.info(
f"{session.short}: <- media-status - audio: {has_audio}, video: {has_video}"
)
session.has_audio = has_audio
session.has_video = has_video
@ -283,18 +293,19 @@ async def websocket_lobby(
await part(lobby, session)
case "relayICECandidate":
if id not in lobby.sessions:
logger.info(data)
if session.id not in lobby.sessions:
logger.error(
f"{session.short}:{session.name} <- relayICECandidate - Does not have Audio"
)
return
peer_id = data.peer_id
candidate = data.candidate
peer_id = data.get("config", {}).get("peer_id")
candidate = data.get("config", {}).get("candidate")
message = {
type: "iceCandidate",
data: {"peer_id": session.id, "candidate": candidate},
"type": "iceCandidate",
"data": {"peer_id": session.id, "candidate": candidate},
}
if peer_id in lobby.sessions:
@ -312,13 +323,14 @@ async def websocket_lobby(
logger.error(
f"{session.short}:{session.name} - relaySessionDescription - Does not have Audio"
)
peer_id = data.peer_id
session_description = data.session_description
peer_id = data.get("config", {}).get("peer_id")
session_description = data.get("config", {}).get(
"session_description"
)
message = {
type: "sessionDescription",
data: {
"peer_id": session.name,
"type": "sessionDescription",
"data": {
"peer_id": session.id,
"session_description": session_description,
},
}
@ -341,6 +353,12 @@ async def websocket_lobby(
except WebSocketDisconnect:
logger.info(f"WebSocket disconnected for user {session_id}")
# Cleanup: remove session from lobby and sessions dict
session.ws = None
if lobby and session:
await part(lobby, session)
# if session_id in sessions:
# del sessions[session_id]
# Serve static files or proxy to frontend development server
@ -355,7 +373,6 @@ if PRODUCTION:
else:
logger.info(f"Proxying static files to http://static-frontend:3000 at {public_url}")
import ssl
@ -385,7 +402,8 @@ else:
filtered_headers = {
k: v
for k, v in proxy_resp.headers.items()
if k.lower() not in ["content-encoding", "transfer-encoding", "content-length"]
if k.lower()
not in ["content-encoding", "transfer-encoding", "content-length"]
}
return Response(
content=content,
@ -405,7 +423,7 @@ else:
async def websocket_proxy(websocket: StarletteWebSocket):
logger.info("WebSocket proxy connection established.")
# Get scheme from websocket.url (should be 'ws' or 'wss')
scheme = websocket.url.scheme if hasattr(websocket, 'url') else 'ws'
scheme = websocket.url.scheme if hasattr(websocket, "url") else "ws"
target_url = f"{scheme}://static-frontend:3000/ws"
await websocket.accept()
try: