From c270c522f3e6461ed08ce9c1f309251078bbd1fa Mon Sep 17 00:00:00 2001 From: James Ketrenos Date: Tue, 16 Sep 2025 12:25:04 -0700 Subject: [PATCH] Improving UX --- client/src/MediaControl.css | 2 +- client/src/MediaControl.tsx | 125 ++++++++++++++++++--------- client/src/UserList.tsx | 36 ++++++++ server/core/lobby_manager.py | 44 +++++++++- server/websocket/message_handlers.py | 30 +++++++ shared/models.py | 2 + 6 files changed, 193 insertions(+), 46 deletions(-) diff --git a/client/src/MediaControl.css b/client/src/MediaControl.css index 4c8cea7..145e438 100644 --- a/client/src/MediaControl.css +++ b/client/src/MediaControl.css @@ -75,7 +75,7 @@ bottom: 0; flex-direction: column; z-index: 1; - align-items: center; + align-items: flex-start; justify-content: center } diff --git a/client/src/MediaControl.tsx b/client/src/MediaControl.tsx index ea241a9..62e408b 100644 --- a/client/src/MediaControl.tsx +++ b/client/src/MediaControl.tsx @@ -1147,11 +1147,11 @@ const MediaAgent = (props: MediaAgentProps) => { if (!updated[session.id]) { // Disable tracks based on initial muted state before assigning to peer if (media) { - media.getAudioTracks().forEach(track => { + media.getAudioTracks().forEach((track) => { track.enabled = false; // Start muted console.log(`media-agent - Local audio track ${track.id} disabled (initial state)`); }); - media.getVideoTracks().forEach(track => { + media.getVideoTracks().forEach((track) => { track.enabled = false; // Start with video off console.log(`media-agent - Local video track ${track.id} disabled (initial state)`); }); @@ -1324,9 +1324,19 @@ interface MediaControlProps { isSelf: boolean; peer: Peer; className?: string; + sendJsonMessage?: (msg: any) => void; + remoteAudioMuted?: boolean; + remoteVideoOff?: boolean; } -const MediaControl: React.FC = ({ isSelf, peer, className }) => { +const MediaControl: React.FC = ({ + isSelf, + peer, + className, + sendJsonMessage, + remoteAudioMuted, + remoteVideoOff, +}) => { const [muted, setMuted] = useState(peer?.muted || false); const [videoOn, setVideoOn] = useState(peer?.video_on !== false); const [isValid, setIsValid] = useState(false); @@ -1340,18 +1350,19 @@ const MediaControl: React.FC = ({ isSelf, peer, className }) const spacerRef = useRef(null); const moveableRef = useRef(null); const [isDragging, setIsDragging] = useState(false); - // Reset drag state on pointerup/touchend/mouseup anywhere in the document - useEffect(() => { - const resetDrag = () => setIsDragging(false); - document.addEventListener("pointerup", resetDrag); - document.addEventListener("touchend", resetDrag); - document.addEventListener("mouseup", resetDrag); - return () => { - document.removeEventListener("pointerup", resetDrag); - document.removeEventListener("touchend", resetDrag); - document.removeEventListener("mouseup", resetDrag); - }; - }, []); + // Get sendJsonMessage from props + // Reset drag state on pointerup/touchend/mouseup anywhere in the document + useEffect(() => { + const resetDrag = () => setIsDragging(false); + document.addEventListener("pointerup", resetDrag); + document.addEventListener("touchend", resetDrag); + document.addEventListener("mouseup", resetDrag); + return () => { + document.removeEventListener("pointerup", resetDrag); + document.removeEventListener("touchend", resetDrag); + document.removeEventListener("mouseup", resetDrag); + }; + }, []); useEffect(() => { console.log( @@ -1502,36 +1513,49 @@ const MediaControl: React.FC = ({ isSelf, peer, className }) const toggleMute = useCallback( (e: React.MouseEvent | React.TouchEvent) => { e.stopPropagation(); - if (peer) { - const newMutedState = !muted; - // Update local state first - setMuted(newMutedState); - // Update peer object (this should trigger re-renders in parent components) - peer.muted = newMutedState; - console.log(`media-agent - toggleMute: ${peer.peer_name} muted=${newMutedState}`); + if (!peer) return; + const newMutedState = !muted; + setMuted(newMutedState); + peer.muted = newMutedState; + console.log(`media-agent - toggleMute: ${peer.peer_name} muted=${newMutedState}`); + // Only broadcast if this is the local user (isSelf) + if (isSelf && sendJsonMessage) { + sendJsonMessage({ + type: "peer_state_update", + data: { + peer_id: peer.session_id, + muted: newMutedState, + video_on: videoOn, + }, + }); } }, - [peer, muted] + [peer, muted, videoOn, sendJsonMessage, isSelf] ); const toggleVideo = useCallback( (e: React.MouseEvent | React.TouchEvent) => { e.stopPropagation(); - if (peer) { - const newVideoState = !videoOn; - // Update local state first - setVideoOn(newVideoState); - // Update peer object (this should trigger re-renders in parent components) - peer.video_on = newVideoState; - console.log(`media-agent - toggleVideo: ${peer.peer_name} video_on=${newVideoState}`); + if (!peer) return; + const newVideoState = !videoOn; + setVideoOn(newVideoState); + peer.video_on = newVideoState; + console.log(`media-agent - toggleVideo: ${peer.peer_name} video_on=${newVideoState}`); + // Only broadcast if this is the local user (isSelf) + if (isSelf && sendJsonMessage) { + sendJsonMessage({ + type: "peer_state_update", + data: { + peer_id: peer.session_id, + muted: muted, + video_on: newVideoState, + }, + }); } }, - [peer, videoOn] + [peer, videoOn, muted, sendJsonMessage, isSelf] ); - // Handlers for bot consent prompts (local user only) - // No bot consent handlers — policy: do not override local mute/video state. - // Snap-back functionality const checkSnapBack = (x: number, y: number) => { if (!spacerRef.current) return false; @@ -1616,13 +1640,28 @@ const MediaControl: React.FC = ({ isSelf, peer, className }) {muted ? : } ) : ( - - {muted ? : } - + + + {muted ? : } + + {remoteAudioMuted && } + )} - - {videoOn ? : } - + + + {videoOn ? : } + + {!isSelf && remoteVideoOff && } + {isValid ? ( peer.attributes?.srcObject && ( @@ -1664,10 +1703,10 @@ const MediaControl: React.FC = ({ isSelf, peer, className }) edge onDragStart={(e) => { // Only block drag if the event is a pointerdown/touchstart on a button, but do not interfere with click/touch events - const controls = containerRef.current?.querySelector('.Controls'); + const controls = containerRef.current?.querySelector(".Controls"); const target = e.inputEvent?.target as HTMLElement; - if (controls && target && (target.closest('button') || target.closest('.MuiIconButton-root'))) { - if (typeof e.stopDrag === 'function') { + if (controls && target && (target.closest("button") || target.closest(".MuiIconButton-root"))) { + if (typeof e.stopDrag === "function") { e.stopDrag(); } return; diff --git a/client/src/UserList.tsx b/client/src/UserList.tsx index 28a8bd5..81935ea 100644 --- a/client/src/UserList.tsx +++ b/client/src/UserList.tsx @@ -26,6 +26,8 @@ type User = { bot_run_id?: string; bot_provider_id?: string; bot_instance_id?: string; // For bot instances + muted?: boolean; + video_on?: boolean; }; type UserListProps = { @@ -127,6 +129,21 @@ const UserList: React.FC = (props: UserListProps) => { }); lobby_state.participants.sort(sortUsers); setUsers(lobby_state.participants); + // Initialize peers with remote mute/video state + setPeers((prevPeers) => { + const updated: Record = { ...prevPeers }; + lobby_state.participants.forEach((user) => { + // Only update remote peers, never overwrite local peer object + if (!user.local && updated[user.session_id]) { + updated[user.session_id] = { + ...updated[user.session_id], + muted: user.muted ?? false, + video_on: user.video_on ?? true, + }; + } + }); + return updated; + }); break; } case "update_name": { @@ -136,6 +153,22 @@ const UserList: React.FC = (props: UserListProps) => { } break; } + case "peer_state_update": { + // Update peer state in peers, but do not override local mute + setPeers((prevPeers) => { + const updated = { ...prevPeers }; + const peerId = data.peer_id; + if (peerId && updated[peerId]) { + updated[peerId] = { + ...updated[peerId], + muted: data.muted, + video_on: data.video_on, + }; + } + return updated; + }); + break; + } default: break; } @@ -255,6 +288,9 @@ const UserList: React.FC = (props: UserListProps) => { key={user.session_id} peer={peers[user.session_id]} isSelf={user.local} + sendJsonMessage={user.local ? sendJsonMessage : undefined} + remoteAudioMuted={peers[user.session_id].muted} + remoteVideoOff={peers[user.session_id].video_on === false} /> ) : user.name && user.live && user.has_media === false ? (
None: + """Broadcast an arbitrary JSON message to all connected sessions in the lobby""" + failed_sessions: List[Session] = [] + for peer in self.sessions.values(): + if peer.ws: + try: + await peer.ws.send_json(message) + except Exception as e: + logger.warning( + f"Failed to send broadcast_json message to {peer.getName()}: {e}" + ) + failed_sessions.append(peer) + for failed_session in failed_sessions: + failed_session.ws = None + async def broadcast_peer_state_update(self, update: dict) -> None: + """Broadcast a peer state update to all connected sessions in the lobby""" + failed_sessions: List[Session] = [] + for peer in self.sessions.values(): + if peer.ws: + try: + await peer.ws.send_json(update) + except Exception as e: + logger.warning( + f"Failed to send peer state update to {peer.getName()}: {e}" + ) + failed_sessions.append(peer) + for failed_session in failed_sessions: + failed_session.ws = None """Individual lobby representing a chat/voice room""" def __init__(self, name: str, id: Optional[str] = None, private: bool = False): @@ -106,7 +134,14 @@ class Lobby: { "type": "lobby_state", "data": { - "participants": [user.model_dump() for user in users] + "participants": [ + { + **user.model_dump(), + "muted": getattr(s, "muted", False), + "video_on": getattr(s, "video_on", True), + } + for user, s in zip(users, self.sessions.values()) + ] }, } ) @@ -130,7 +165,12 @@ class Lobby: "type": "lobby_state", "data": { "participants": [ - user.model_dump() for user in users + { + **user.model_dump(), + "muted": getattr(s, "muted", False), + "video_on": getattr(s, "video_on", True), + } + for user, s in zip(users, self.sessions.values()) ] }, } diff --git a/server/websocket/message_handlers.py b/server/websocket/message_handlers.py index 477508a..2951ce2 100644 --- a/server/websocket/message_handlers.py +++ b/server/websocket/message_handlers.py @@ -40,6 +40,35 @@ class MessageHandler(ABC): """Handle a WebSocket message""" pass +class PeerStateUpdateHandler(MessageHandler): + """Handler for peer_state_update messages""" + + async def handle( + self, + session: Any, + lobby: Any, + data: dict, + websocket: Any, + managers: dict, + ) -> None: + # Only allow a user to update their own state + if not lobby or not session: + return + peer_id = data.get("peer_id", getattr(session, "id", None)) + if str(peer_id) != str(getattr(session, "id", None)): + # Ignore attempts to update other users' state + # Optionally log or send error to client + return + update = { + "type": "peer_state_update", + "data": { + "peer_id": peer_id, + "muted": data.get("muted"), + "video_on": data.get("video_on"), + }, + } + await lobby.broadcast_peer_state_update(update) + class SetNameHandler(MessageHandler): """Handler for set_name messages""" @@ -413,6 +442,7 @@ class MessageRouter: # Bot monitoring handlers self.register("status_check", StatusCheckHandler()) + self.register("peer_state_update", PeerStateUpdateHandler()) def register(self, message_type: str, handler: MessageHandler): """Register a handler for a message type""" diff --git a/shared/models.py b/shared/models.py index 241d244..f8e71d2 100644 --- a/shared/models.py +++ b/shared/models.py @@ -47,6 +47,8 @@ class ParticipantModel(BaseModel): bot_run_id: Optional[str] = None bot_provider_id: Optional[str] = None bot_instance_id: Optional[str] = None + muted: bool = False + video_on: bool = True # =============================================================================