jketreno/ai-voicebot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improving UX

Browse Source
This commit is contained in:
James Ketr 2025-09-16 12:25:04 -07:00
parent 77a3bf89a7
commit c270c522f3
6 changed files with 193 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
client/src/MediaControl.css
View File

@ -75,7 +75,7 @@
bottom: 0;
flex-direction: column;
z-index: 1;
align-items: center;
align-items: flex-start;
justify-content: center
}

125
client/src/MediaControl.tsx
View File

@ -1147,11 +1147,11 @@ const MediaAgent = (props: MediaAgentProps) => {
if (!updated[session.id]) {
// Disable tracks based on initial muted state before assigning to peer
if (media) {
media.getAudioTracks().forEach(track => {
media.getAudioTracks().forEach((track) => {
track.enabled = false; // Start muted
console.log(`media-agent - Local audio track ${track.id} disabled (initial state)`);
});
media.getVideoTracks().forEach(track => {
media.getVideoTracks().forEach((track) => {
track.enabled = false; // Start with video off
console.log(`media-agent - Local video track ${track.id} disabled (initial state)`);
});
@ -1324,9 +1324,19 @@ interface MediaControlProps {
isSelf: boolean;
peer: Peer;
className?: string;
sendJsonMessage?: (msg: any) => void;
remoteAudioMuted?: boolean;
remoteVideoOff?: boolean;
}
const MediaControl: React.FC<MediaControlProps> = ({ isSelf, peer, className }) => {
const MediaControl: React.FC<MediaControlProps> = ({
isSelf,
peer,
className,
sendJsonMessage,
remoteAudioMuted,
remoteVideoOff,
}) => {
const [muted, setMuted] = useState<boolean>(peer?.muted || false);
const [videoOn, setVideoOn] = useState<boolean>(peer?.video_on !== false);
const [isValid, setIsValid] = useState<boolean>(false);
@ -1340,18 +1350,19 @@ const MediaControl: React.FC<MediaControlProps> = ({ isSelf, peer, className })
const spacerRef = useRef<HTMLDivElement>(null);
const moveableRef = useRef<any>(null);
const [isDragging, setIsDragging] = useState<boolean>(false);
// Reset drag state on pointerup/touchend/mouseup anywhere in the document
useEffect(() => {
const resetDrag = () => setIsDragging(false);
document.addEventListener("pointerup", resetDrag);
document.addEventListener("touchend", resetDrag);
document.addEventListener("mouseup", resetDrag);
return () => {
document.removeEventListener("pointerup", resetDrag);
document.removeEventListener("touchend", resetDrag);
document.removeEventListener("mouseup", resetDrag);
};
}, []);
// Get sendJsonMessage from props
// Reset drag state on pointerup/touchend/mouseup anywhere in the document
useEffect(() => {
const resetDrag = () => setIsDragging(false);
document.addEventListener("pointerup", resetDrag);
document.addEventListener("touchend", resetDrag);
document.addEventListener("mouseup", resetDrag);
return () => {
document.removeEventListener("pointerup", resetDrag);
document.removeEventListener("touchend", resetDrag);
document.removeEventListener("mouseup", resetDrag);
};
}, []);
useEffect(() => {
console.log(
@ -1502,36 +1513,49 @@ const MediaControl: React.FC<MediaControlProps> = ({ isSelf, peer, className })
const toggleMute = useCallback(
(e: React.MouseEvent | React.TouchEvent) => {
e.stopPropagation();
if (peer) {
const newMutedState = !muted;
// Update local state first
setMuted(newMutedState);
// Update peer object (this should trigger re-renders in parent components)
peer.muted = newMutedState;
console.log(`media-agent - toggleMute: ${peer.peer_name} muted=${newMutedState}`);
if (!peer) return;
const newMutedState = !muted;
setMuted(newMutedState);
peer.muted = newMutedState;
console.log(`media-agent - toggleMute: ${peer.peer_name} muted=${newMutedState}`);
// Only broadcast if this is the local user (isSelf)
if (isSelf && sendJsonMessage) {
sendJsonMessage({
type: "peer_state_update",
data: {
peer_id: peer.session_id,
muted: newMutedState,
video_on: videoOn,
},
});
}
},
[peer, muted]
[peer, muted, videoOn, sendJsonMessage, isSelf]
);
const toggleVideo = useCallback(
(e: React.MouseEvent | React.TouchEvent) => {
e.stopPropagation();
if (peer) {
const newVideoState = !videoOn;
// Update local state first
setVideoOn(newVideoState);
// Update peer object (this should trigger re-renders in parent components)
peer.video_on = newVideoState;
console.log(`media-agent - toggleVideo: ${peer.peer_name} video_on=${newVideoState}`);
if (!peer) return;
const newVideoState = !videoOn;
setVideoOn(newVideoState);
peer.video_on = newVideoState;
console.log(`media-agent - toggleVideo: ${peer.peer_name} video_on=${newVideoState}`);
// Only broadcast if this is the local user (isSelf)
if (isSelf && sendJsonMessage) {
sendJsonMessage({
type: "peer_state_update",
data: {
peer_id: peer.session_id,
muted: muted,
video_on: newVideoState,
},
});
}
},
[peer, videoOn]
[peer, videoOn, muted, sendJsonMessage, isSelf]
);
// Handlers for bot consent prompts (local user only)
// No bot consent handlers — policy: do not override local mute/video state.
// Snap-back functionality
const checkSnapBack = (x: number, y: number) => {
if (!spacerRef.current) return false;
@ -1616,13 +1640,28 @@ const MediaControl: React.FC<MediaControlProps> = ({ isSelf, peer, className })
{muted ? <MicOff color={colorAudio} /> : <Mic color={colorAudio} />}
</IconButton>
) : (
<IconButton onClick={toggleMute}>
{muted ? <VolumeOff color={colorAudio} /> : <VolumeUp color={colorAudio} />}
</IconButton>
<Box sx={{ display: "flex", flexDirection: "row", gap: 0, alignItems: "center", p: 0, m: 0 }}>
<IconButton onClick={toggleMute}>
{muted ? <VolumeOff color={colorAudio} /> : <VolumeUp color={colorAudio} />}
</IconButton>
{remoteAudioMuted && <MicOff color="warning" />}
</Box>
)}
<IconButton onClick={toggleVideo}>
{videoOn ? <Videocam color={colorVideo} /> : <VideocamOff color={colorVideo} />}
</IconButton>
<Box
sx={{
display: "flex",
flexDirection: "row",
gap: 0,
alignItems: "center",
p: 0,
m: 0,
}}
>
<IconButton onClick={toggleVideo}>
{videoOn ? <Videocam color={colorVideo} /> : <VideocamOff color={colorVideo} />}
</IconButton>
{!isSelf && remoteVideoOff && <VideocamOff color="warning" />}
</Box>
</Box>
{isValid ? (
peer.attributes?.srcObject && (
@ -1664,10 +1703,10 @@ const MediaControl: React.FC<MediaControlProps> = ({ isSelf, peer, className })
edge
onDragStart={(e) => {
// Only block drag if the event is a pointerdown/touchstart on a button, but do not interfere with click/touch events
const controls = containerRef.current?.querySelector('.Controls');
const controls = containerRef.current?.querySelector(".Controls");
const target = e.inputEvent?.target as HTMLElement;
if (controls && target && (target.closest('button') || target.closest('.MuiIconButton-root'))) {
if (typeof e.stopDrag === 'function') {
if (controls && target && (target.closest("button") || target.closest(".MuiIconButton-root"))) {
if (typeof e.stopDrag === "function") {
e.stopDrag();
}
return;

36
client/src/UserList.tsx
View File

@ -26,6 +26,8 @@ type User = {
bot_run_id?: string;
bot_provider_id?: string;
bot_instance_id?: string; // For bot instances
muted?: boolean;
video_on?: boolean;
};
type UserListProps = {
@ -127,6 +129,21 @@ const UserList: React.FC<UserListProps> = (props: UserListProps) => {
});
lobby_state.participants.sort(sortUsers);
setUsers(lobby_state.participants);
// Initialize peers with remote mute/video state
setPeers((prevPeers) => {
const updated: Record<string, Peer> = { ...prevPeers };
lobby_state.participants.forEach((user) => {
// Only update remote peers, never overwrite local peer object
if (!user.local && updated[user.session_id]) {
updated[user.session_id] = {
...updated[user.session_id],
muted: user.muted ?? false,
video_on: user.video_on ?? true,
};
}
});
return updated;
});
break;
}
case "update_name": {
@ -136,6 +153,22 @@ const UserList: React.FC<UserListProps> = (props: UserListProps) => {
}
break;
}
case "peer_state_update": {
// Update peer state in peers, but do not override local mute
setPeers((prevPeers) => {
const updated = { ...prevPeers };
const peerId = data.peer_id;
if (peerId && updated[peerId]) {
updated[peerId] = {
...updated[peerId],
muted: data.muted,
video_on: data.video_on,
};
}
return updated;
});
break;
}
default:
break;
}
@ -255,6 +288,9 @@ const UserList: React.FC<UserListProps> = (props: UserListProps) => {
key={user.session_id}
peer={peers[user.session_id]}
isSelf={user.local}
sendJsonMessage={user.local ? sendJsonMessage : undefined}
remoteAudioMuted={peers[user.session_id].muted}
remoteVideoOff={peers[user.session_id].video_on === false}
/>
) : user.name && user.live && user.has_media === false ? (
<div

44
server/core/lobby_manager.py
View File

@ -62,6 +62,34 @@ class LobbyConfig:
class Lobby:
async def broadcast_json(self, message: dict) -> None:
"""Broadcast an arbitrary JSON message to all connected sessions in the lobby"""
failed_sessions: List[Session] = []
for peer in self.sessions.values():
if peer.ws:
try:
await peer.ws.send_json(message)
except Exception as e:
logger.warning(
f"Failed to send broadcast_json message to {peer.getName()}: {e}"
)
failed_sessions.append(peer)
for failed_session in failed_sessions:
failed_session.ws = None
async def broadcast_peer_state_update(self, update: dict) -> None:
"""Broadcast a peer state update to all connected sessions in the lobby"""
failed_sessions: List[Session] = []
for peer in self.sessions.values():
if peer.ws:
try:
await peer.ws.send_json(update)
except Exception as e:
logger.warning(
f"Failed to send peer state update to {peer.getName()}: {e}"
)
failed_sessions.append(peer)
for failed_session in failed_sessions:
failed_session.ws = None
"""Individual lobby representing a chat/voice room"""
def __init__(self, name: str, id: Optional[str] = None, private: bool = False):
@ -106,7 +134,14 @@ class Lobby:
{
"type": "lobby_state",
"data": {
"participants": [user.model_dump() for user in users]
"participants": [
{
**user.model_dump(),
"muted": getattr(s, "muted", False),
"video_on": getattr(s, "video_on", True),
}
for user, s in zip(users, self.sessions.values())
]
},
}
)
@ -130,7 +165,12 @@ class Lobby:
"type": "lobby_state",
"data": {
"participants": [
user.model_dump() for user in users
{
**user.model_dump(),
"muted": getattr(s, "muted", False),
"video_on": getattr(s, "video_on", True),
}
for user, s in zip(users, self.sessions.values())
]
},
}

30
server/websocket/message_handlers.py
View File

@ -40,6 +40,35 @@ class MessageHandler(ABC):
"""Handle a WebSocket message"""
pass
class PeerStateUpdateHandler(MessageHandler):
"""Handler for peer_state_update messages"""
async def handle(
self,
session: Any,
lobby: Any,
data: dict,
websocket: Any,
managers: dict,
) -> None:
# Only allow a user to update their own state
if not lobby or not session:
return
peer_id = data.get("peer_id", getattr(session, "id", None))
if str(peer_id) != str(getattr(session, "id", None)):
# Ignore attempts to update other users' state
# Optionally log or send error to client
return
update = {
"type": "peer_state_update",
"data": {
"peer_id": peer_id,
"muted": data.get("muted"),
"video_on": data.get("video_on"),
},
}
await lobby.broadcast_peer_state_update(update)
class SetNameHandler(MessageHandler):
"""Handler for set_name messages"""
@ -413,6 +442,7 @@ class MessageRouter:
# Bot monitoring handlers
self.register("status_check", StatusCheckHandler())
self.register("peer_state_update", PeerStateUpdateHandler())
def register(self, message_type: str, handler: MessageHandler):
"""Register a handler for a message type"""

2
shared/models.py
View File

@ -47,6 +47,8 @@ class ParticipantModel(BaseModel):
bot_run_id: Optional[str] = None
bot_provider_id: Optional[str] = None
bot_instance_id: Optional[str] = None
muted: bool = False
video_on: bool = True
# =============================================================================