ai-voicebot/client/src/MediaControl.tsx

741 lines
25 KiB
TypeScript

import React, { useState, useEffect, useRef, useCallback, useContext } from "react";
import Moveable from "react-moveable";
import "./MediaControl.css";
import VolumeOff from "@mui/icons-material/VolumeOff";
import VolumeUp from "@mui/icons-material/VolumeUp";
import MicOff from "@mui/icons-material/MicOff";
import Mic from "@mui/icons-material/Mic";
import VideocamOff from "@mui/icons-material/VideocamOff";
import Videocam from "@mui/icons-material/Videocam";
import { GlobalContext } from "./GlobalContext";
import Box from "@mui/material/Box";
const debug = true;
// Types for peer and track context
interface Peer {
sessionId: string;
peerName: string;
hasAudio: boolean;
hasVideo: boolean;
attributes: Record<string, any>;
muted: boolean;
videoOn: boolean;
local: boolean;
dead: boolean;
connection?: RTCPeerConnection;
}
interface TrackContext {
media: MediaStream;
audio: boolean;
video: boolean;
}
interface AddPeerConfig {
peer_id: string;
peer_name: string;
hasAudio: boolean;
hasVideo: boolean;
should_create_offer?: boolean;
}
interface SessionDescriptionData {
peer_id: string;
session_description: RTCSessionDescriptionInit;
}
interface IceCandidateData {
peer_id: string;
candidate: RTCIceCandidateInit;
}
interface RemovePeerData {
peer_id: string;
}
interface VideoProps extends React.VideoHTMLAttributes<HTMLVideoElement> {
srcObject: MediaProvider;
local?: boolean;
}
const Video: React.FC<VideoProps> = ({ srcObject, local, ...props }) => {
const refVideo = useRef<HTMLVideoElement>(null);
useEffect(() => {
if (!refVideo.current) {
return;
}
const ref = refVideo.current;
if (debug) console.log("media-control - video <video> bind");
ref.srcObject = srcObject;
if (local) {
ref.muted = true;
}
return () => {
if (debug) console.log("media-control - <video> unbind");
if (ref) {
(ref as any).srcObject = undefined;
}
};
}, [srcObject, local]);
return <video ref={refVideo} {...props} />;
};
type MediaAgentProps = {
setPeers: (peers: Record<string, Peer>) => void;
};
const MediaAgent = (props: MediaAgentProps) => {
const { setPeers } = props;
const { name, ws, sessionId } = useContext(GlobalContext);
const [peers] = useState<Record<string, Peer>>({});
const [track, setTrack] = useState<TrackContext | undefined>(undefined);
const ignore = useRef(false);
const onTrack = useCallback(
(event: RTCTrackEvent) => {
const connection = event.target as RTCPeerConnection;
console.log("media-agent - ontrack", event);
for (let peer in peers) {
if (peers[peer].connection === connection) {
console.log(`media-agent - ontrack - remote ${peer} track assigned.`);
Object.assign(peers[peer].attributes, {
srcObject: event.streams[0] || event.track,
});
/* Trigger update of MediaControl now that a track is available */
setPeers(Object.assign({}, peers));
}
}
},
[peers, setPeers]
);
const refOnTrack = useRef(onTrack);
const sendMessage = useCallback(
(data: LobbyMessage) => {
if (!ws) {
return;
}
ws.send(JSON.stringify(data));
},
[ws]
);
const onWsMessage = useCallback(
(event: MessageEvent) => {
const addPeer = (config: AddPeerConfig) => {
console.log("media-agent - Signaling server said to add peer:", config);
if (!track) {
console.log(`media-agent - No local media track`);
return;
}
const { peer_id, peer_name } = config;
if (peer_id in peers) {
if (!peers[peer_id].dead) {
/* This is normal when peers are added by other connecting
* peers through the signaling server */
console.log(`media-agent - addPeer - ${peer_id} already in peers`);
return;
}
}
/* Even if reviving, allocate a new Object so <MediaControl> will
* have its peer state change and trigger an update from
* <PlayerList> */
const peer: Peer = {
sessionId: peer_id,
peerName: peer_name,
hasAudio: config.hasAudio,
hasVideo: config.hasVideo,
attributes: {},
muted: false,
videoOn: true,
local: false,
dead: false,
};
if (peer_id in peers) {
peer.muted = peers[peer_id].muted;
peer.videoOn = peers[peer_id].videoOn;
console.log(`media-agent - addPeer - reviving dead peer ${peer_id}`, peer);
} else {
peer.muted = false;
peer.videoOn = true;
}
peers[peer_id] = peer;
console.log(`media-agent - addPeer - remote`, peers);
setPeers(Object.assign({}, peers));
// RTCPeerConnection config should be passed directly, not as 'configuration' property
const connection = new RTCPeerConnection({
iceServers: [
{
urls: "turns:ketrenos.com:5349",
username: "ketra",
credential: "ketran",
},
],
});
peer.connection = connection;
connection.addEventListener("connectionstatechange", (event) => {
console.log(`media-agent - connectionstatechange - `, connection.connectionState, event);
});
connection.addEventListener("negotiationneeded", (event) => {
console.log(`media-agent - negotiationneeded - `, connection.connectionState, event);
});
connection.addEventListener("icecandidateerror", (event: RTCPeerConnectionIceErrorEvent) => {
if (event.errorCode === 701) {
if (connection.iceGatheringState === "gathering") {
console.log(`media-agent - Unable to reach host: ${event.url}`);
} else {
// hostcandidate is deprecated and not always present
console.error(
`media-agent - icecandidateerror - `,
event.errorCode,
(event as any).hostcandidate,
event.url,
event.errorText
);
}
}
});
connection.onicecandidate = (event: RTCPeerConnectionIceEvent) => {
if (!event.candidate) {
console.log(`media-agent - icecanditate - gathering is complete: ${connection.connectionState}`);
return;
}
// If a srflx candidate was found, notify that the STUN server works!
if (event.candidate && event.candidate.type === "srflx") {
console.log("media-agent - The STUN server is reachable!");
// address is not standard, use candidate.candidate string parsing if needed
}
// If a relay candidate was found, notify that the TURN server works!
if (event.candidate && event.candidate.type === "relay") {
console.log("media-agent - The TURN server is reachable !");
}
console.log(`media-agent - onicecandidate - `, event.candidate);
sendMessage({
type: "relayICECandidate",
config: {
peer_id,
candidate: event.candidate,
},
});
};
connection.ontrack = (e: RTCTrackEvent) => refOnTrack.current(e);
// Add all tracks from local media
track.media.getTracks().forEach((t) => {
connection.addTrack(t, track.media);
});
/* Only one side of the peer connection should create the
* offer, the signaling server picks one to be the offerer.
* The other user will get a 'sessionDescription' event and will
* create an offer, then send back an answer 'sessionDescription'
* to us
*/
if (config.should_create_offer) {
if (debug) console.log(`media-agent - Creating RTC offer to ${peer_id}`);
connection
.createOffer()
.then((local_description) => {
if (debug) console.log(`media-agent - Local offer description is: `, local_description);
return connection.setLocalDescription(local_description).then(() => {
sendMessage({
type: "relaySessionDescription",
config: {
peer_id,
session_description: local_description,
},
});
if (debug) console.log(`media-agent - Offer setLocalDescription succeeded`);
});
})
.catch((error) => {
console.error(`media-agent - Offer setLocalDescription failed!`, error);
});
}
};
const sessionDescription = ({ peer_id, session_description }: SessionDescriptionData) => {
const peer = peers[peer_id];
if (!peer) {
console.error(`media-agent - sessionDescription - No peer for ${peer_id}`);
return;
}
const { connection } = peer;
if (!connection) {
console.error(`media-agent - sessionDescription - No connection for peer ${peer_id}`);
return;
}
const desc = new RTCSessionDescription(session_description);
connection
.setRemoteDescription(desc)
.then(() => {
if (debug) console.log(`media-agent - sessionDescription - setRemoteDescription succeeded`);
if (session_description.type === "offer") {
if (debug) console.log(`media-agent - sessionDescription - Creating answer`);
connection
.createAnswer()
.then((local_description) => {
if (debug)
console.log(`media-agent - sessionDescription - Answer description is: `, local_description);
connection
.setLocalDescription(local_description)
.then(() => {
sendMessage({
type: "relaySessionDescription",
config: {
peer_id,
session_description: local_description,
},
});
if (debug) console.log(`media-agent - sessionDescription - Answer setLocalDescription succeeded`);
})
.catch(() => {
console.error(`media-agent - sessionDescription - Answer setLocalDescription failed!`);
});
})
.catch((error) => {
console.error(error);
});
}
})
.catch((error) => {
console.log(`media-agent - sessionDescription - setRemoteDescription error: `, error);
});
};
const removePeer = ({ peer_id }: RemovePeerData) => {
console.log(`media-agent - removePeer - Signaling server said to ` + `remove peer ${peer_id}`);
if (peer_id in peers) {
if (peers[peer_id].connection) {
peers[peer_id].connection.close();
peers[peer_id].connection = undefined;
}
}
/* To maintain mute/videoOn states, we don't remove the peer but
* instead mark it as dead */
peers[peer_id].dead = true;
if (debug) console.log(`media-agent - removePeer`, peers);
setPeers(Object.assign({}, peers));
};
const iceCandidate = ({ peer_id, candidate }: IceCandidateData) => {
/**
* The offerer will send a number of ICE Candidate blobs to the
* answerer so they can begin trying to find the best path to one
* another on the net.
*/
const peer = peers[peer_id];
if (!peer) {
console.error(`media-agent - iceCandidate - No peer for ${peer_id}`, peers);
return;
}
peer.connection
?.addIceCandidate(new RTCIceCandidate(candidate))
.then(() => {
if (debug) console.log(`media-agent - iceCandidate - Successfully added Ice Candidate for ${peer_id}`);
})
.catch((error) => {
console.error(error, peer, candidate);
});
};
const data = JSON.parse(event.data);
if (["addPeer", "removePeer", "iceCandidate", "sessionDescription"].includes(data.type)) {
console.log(`media-agent - message - ${data.type}`, peers);
}
switch (data.type) {
case "addPeer":
addPeer(data.data);
break;
case "removePeer":
removePeer(data.data);
break;
case "iceCandidate":
iceCandidate(data.data);
break;
case "sessionDescription":
sessionDescription(data.data);
break;
default:
break;
}
},
[peers, setPeers, track, refOnTrack, sendMessage]
);
const refWsMessage = useRef(onWsMessage);
const onWsClose = (_event: CloseEvent) => {
console.log(`media-agent - ${name} Disconnected from signaling server`);
/* Tear down all of our peer connections and remove all the
* media divs when we disconnect */
for (let peer_id in peers) {
if (peers[peer_id].local) {
continue;
}
if (peers[peer_id].connection) {
peers[peer_id].connection.close();
peers[peer_id].connection = undefined;
}
}
for (let id in peers) {
peers[id].dead = true;
peers[id].connection = undefined;
}
if (debug) console.log(`media-agent - close`, peers);
setPeers(Object.assign({}, peers));
};
const refWsClose = useRef(onWsClose);
useEffect(() => {
refWsMessage.current = onWsMessage;
refWsClose.current = onWsClose;
refOnTrack.current = onTrack;
});
useEffect(() => {
if (!ws) {
return;
}
console.log(`media-control - Binding to WebSocket`);
const cbMessage = (e: MessageEvent) => refWsMessage.current(e);
ws.addEventListener("message", cbMessage);
const cbClose = (e: CloseEvent) => refWsClose.current(e);
ws.addEventListener("close", cbClose);
return () => {
ws.removeEventListener("message", cbMessage);
ws.removeEventListener("close", cbClose);
};
}, [ws, refWsMessage, refWsClose]);
useEffect(() => {
console.log(`media-control - WebSocket or Track changed`);
const join = () => {
sendMessage({
type: "join",
data: {
has_audio: track?.audio ?? false,
has_aideo: track?.video ?? false,
},
});
};
if (ws && track) {
console.log(`media-conterol - issuing join request`);
for (let peer in peers) {
if (peers[peer].local && peers[peer].dead) {
/* Allocate a new Object so <MediaControl> will trigger */
peers[peer] = Object.assign({}, peers[peer]);
// Mark as alive
peers[peer].dead = false;
setPeers(Object.assign({}, peers));
}
}
join();
}
}, [ws, track, peers, setPeers, sendMessage]);
useEffect(() => {
if (!sessionId) {
return;
}
let update = false;
if (track) {
if (!(sessionId in peers)) {
update = true;
peers[sessionId] = {
peerName: name || "Unknown",
sessionId: sessionId,
local: true,
muted: true,
videoOn: false,
hasVideo: track.video,
hasAudio: track.audio,
attributes: {
local: true,
srcObject: track.media,
},
dead: false,
};
}
}
/* Renaming the local connection requires the peer to be deleted
* and re-established with the signaling server */
for (let key in peers) {
if (peers[key].local && key !== sessionId) {
delete peers[key];
update = true;
}
}
if (update) {
if (debug) console.log(`media-agent - Setting global peers`, peers);
setPeers(Object.assign({}, peers));
}
}, [peers, name, setPeers, track]);
useEffect(() => {
if (!ws || !name) {
return;
}
type setup_local_media_props = {
audio?: boolean;
video?: boolean;
}
const setup_local_media = async (
props?: setup_local_media_props
): Promise<TrackContext> => {
const { audio = true, video = true } = props ?? {};
// Ask user for permission to use the computers microphone and/or camera
console.log(
`media-agent - Requesting access to local audio: ${audio} / video: ${video} inputs`
);
try {
const media = await navigator.mediaDevices.getUserMedia({
audio,
video,
});
sendMessage({ type: "media_status", video, audio });
// Optionally apply constraints
if (video && media.getVideoTracks().length > 0) {
console.log(`media-agent - Applying video constraints to ${media.getVideoTracks().length} video tracks`);
media.getVideoTracks().forEach((track) => {
track.applyConstraints({
width: { min: 160, max: 320 },
height: { min: 120, max: 240 },
});
});
}
return { media, audio, video };
} catch (error) {
if (video) {
console.log(`media-agent - Access to audio and video failed. Trying just audio.`);
// Try again with only audio if video failed
return setup_local_media({ audio, video: false });
} else if (audio) {
console.log(`media-agent - Access to audio failed.`);
sendMessage({ type: "media_status", video: false, audio: false });
// Return a dummy context with no media
return { media: new MediaStream(), audio: false, video: false };
} else {
// No media requested or available
sendMessage({ type: "media_status", video: false, audio: false });
return { media: new MediaStream(), audio: false, video: false };
}
}
};
if (!track) {
if (debug) console.log(`media-agent - WebSocket open request. ` + `Attempting to create local media.`);
setup_local_media()
.then((context) => {
console.log(`media-agent - local media setup complete`, context);
/* once the user has given us access to their
* microphone/camcorder, join the channel and start peering up */
console.log(`media-agent - ignore set to ${ignore.current}`);
// if (ignore.current) {
// console.log(`media-agent - aborting setting local media`);
// } else {
console.log("media-agent - setTrack called with context:", context);
setTrack(context);
// }
})
.catch((error) => {
/* user denied access to a/v */
console.error("media-agent - error in setup_local_media", error);
console.log("media-agent - Access denied for audio/video");
});
}
return () => {
ignore.current = true;
if (!track) {
console.log(`media-agent - abort media setup!`);
}
};
}, [ws, track, name, sendMessage]);
return <></>;
};
interface MediaControlProps {
isSelf: boolean;
peer: Peer;
className?: string;
}
const MediaControl: React.FC<MediaControlProps> = ({ isSelf, peer, className }) => {
const [media, setMedia] = useState<Peer | undefined>(undefined);
const [muted, setMuted] = useState<boolean | undefined>(undefined);
const [videoOn, setVideoOn] = useState<boolean | undefined>(undefined);
const [target, setTarget] = useState<Element | undefined>();
const [frame, setFrame] = useState<{ translate: [number, number] }>({
translate: [0, 0],
});
useEffect(() => {
if (peer && peer.peerName) {
const el = document.querySelector(`.MediaControl[data-peer="${peer.sessionId}"]`);
setTarget(el ?? undefined);
}
}, [setTarget, peer]);
/* local state is used to trigger re-renders, and the global
* state is kept up to date in the peers object so re-assignment
* of sessions doesn't kill the peer or change the mute/video states */
useEffect(() => {
if (!peer) {
setMedia(undefined);
return;
}
setMuted(peer.muted);
setVideoOn(peer.videoOn);
setMedia(peer);
}, [peer, setMedia, setMuted, setVideoOn]);
console.log(`media-control - render`);
const toggleMute = (event: React.MouseEvent | React.TouchEvent) => {
if (debug) console.log(`media-control - toggleMute - ${peer.peerName}`, !muted);
if (peer) {
peer.muted = !muted;
setMuted(peer.muted);
}
event.stopPropagation();
};
const toggleVideo = (event: React.MouseEvent | React.TouchEvent) => {
if (debug) console.log(`media-control - toggleVideo - ${peer.peerName}`, !videoOn);
if (peer) {
peer.videoOn = !videoOn;
if (peer.videoOn && media) {
const video = document.querySelector(`video[data-id="${media.peerName}"]`) as HTMLVideoElement | null;
if (video && typeof video.play === "function") {
video.play();
}
}
setVideoOn(peer.videoOn);
}
event.stopPropagation();
};
useEffect(() => {
if (!media || media.dead || !peer) {
return;
}
if (media.attributes.srcObject) {
console.log(`media-control - audio enable - ${peer.peerName}:${!muted}`);
(media.attributes.srcObject.getAudioTracks() as MediaStreamTrack[]).forEach((track: MediaStreamTrack) => {
track.enabled = media.hasAudio && !muted;
});
}
});
useEffect(() => {
if (!media || media.dead || !peer) {
return;
}
if (media.attributes.srcObject) {
console.log(`media-control - video enable - ${peer.peerName}:${videoOn}`);
(media.attributes.srcObject.getVideoTracks() as MediaStreamTrack[]).forEach((track: MediaStreamTrack) => {
track.enabled = Boolean(media.hasVideo) && Boolean(videoOn);
});
}
});
const isValid = media && !media.dead,
colorAudio = isValid && media.hasAudio ? "primary" : "disabled",
colorVideo = isValid && media.hasVideo ? "primary" : "disabled";
if (!peer) {
console.log(`media-control - no peer`);
return <></>;
}
return (
<Box sx={{ display: 'flex', flexDirection: 'column', alignItems: 'center', border: "3px solid green", minWidth: '200px', minHeight: '100px' }}>
<div className={`MediaControlSpacer ${className}`} />
<div className={`MediaControl ${className}`} data-peer={peer.sessionId}>
<div className="Controls">
{isSelf && (
<div onTouchStart={toggleMute} onClick={toggleMute}>
{muted && <MicOff color={colorAudio} />}
{!muted && <Mic color={colorAudio} />}
</div>
)}
{!isSelf && (
<div onTouchStart={toggleMute} onClick={toggleMute}>
{muted && <VolumeOff color={colorAudio} />}
{!muted && <VolumeUp color={colorAudio} />}
</div>
)}
<div onTouchStart={toggleVideo} onClick={toggleVideo}>
{!videoOn && <VideocamOff color={colorVideo} />}
{videoOn && <Videocam color={colorVideo} />}
</div>
</div>
{isValid && (
<>
<Moveable
pinchable={true}
draggable={true}
// Moveable expects HTMLElement or SVGElement, not just Element
target={target as HTMLElement | SVGElement | undefined}
resizable={true}
keepRatio={true}
throttleResize={0}
hideDefaultLines={false}
edge={true}
zoom={1}
origin={false}
onDragStart={(e) => {
e.set(frame.translate as [number, number]);
}}
onDrag={(e) => {
// Defensive: ensure beforeTranslate is [number, number]
if (Array.isArray(e.beforeTranslate) && e.beforeTranslate.length === 2) {
frame.translate = [e.beforeTranslate[0], e.beforeTranslate[1]];
}
}}
onResizeStart={(e) => {
e.setOrigin(["%", "%"]);
e.dragStart && e.dragStart.set(frame.translate as [number, number]);
}}
onResize={(e) => {
const { translate } = frame;
e.target.style.width = `${e.width}px`;
e.target.style.height = `${e.height}px`;
e.target.style.transform = `translate(${translate[0]}px, ${translate[1]}px)`;
}}
onRender={(e) => {
const { translate } = frame;
e.target.style.transform = `translate(${translate[0]}px, ${translate[1]}px)`;
}}
/>
<Video
className="Video"
data-id={media.peerName}
autoPlay={true}
srcObject={media.attributes.srcObject}
{...media.attributes}
/>
</>
)}
{!isValid && <video className="Video"></video>}
</div>
</Box>
);
};
export { MediaControl, MediaAgent };