1233 lines
42 KiB
TypeScript
1233 lines
42 KiB
TypeScript
import React, { useState, useEffect, useRef, useCallback } from "react";
|
|
import Moveable from "react-moveable";
|
|
import { flushSync } from "react-dom";
|
|
import "./MediaControl.css";
|
|
import VolumeOff from "@mui/icons-material/VolumeOff";
|
|
import VolumeUp from "@mui/icons-material/VolumeUp";
|
|
import MicOff from "@mui/icons-material/MicOff";
|
|
import Mic from "@mui/icons-material/Mic";
|
|
import VideocamOff from "@mui/icons-material/VideocamOff";
|
|
import Videocam from "@mui/icons-material/Videocam";
|
|
import Box from "@mui/material/Box";
|
|
import useWebSocket, { ReadyState } from "react-use-websocket";
|
|
import { Session } from "./GlobalContext";
|
|
|
|
const debug = true;
|
|
|
|
/* ---------- Synthetic Tracks Helpers ---------- */
|
|
|
|
// Helper to hash a string to a color
|
|
function nameToColor(name: string): string {
|
|
// Simple hash function (djb2)
|
|
let hash = 5381;
|
|
for (let i = 0; i < name.length; i++) {
|
|
hash = (hash << 5) + hash + name.charCodeAt(i);
|
|
}
|
|
// Generate HSL color from hash
|
|
const hue = Math.abs(hash) % 360;
|
|
const sat = 60 + (Math.abs(hash) % 30); // 60-89%
|
|
const light = 45 + (Math.abs(hash) % 30); // 45-74%
|
|
return `hsl(${hue},${sat}%,${light}%)`;
|
|
}
|
|
|
|
// Accepts an optional name for color selection
|
|
const createAnimatedVideoTrack = ({
|
|
width = 320,
|
|
height = 240,
|
|
name,
|
|
}: { width?: number; height?: number; name?: string } = {}): MediaStreamTrack => {
|
|
const canvas = document.createElement("canvas");
|
|
canvas.width = width;
|
|
canvas.height = height;
|
|
|
|
const ctx = canvas.getContext("2d");
|
|
if (!ctx) throw new Error("Could not get canvas context");
|
|
|
|
// Pick color based on name, fallback to default
|
|
const ballColor = name ? nameToColor(name) : "#00ff88";
|
|
|
|
const ball = {
|
|
x: width / 2,
|
|
y: height / 2,
|
|
radius: Math.min(width, height) * 0.06,
|
|
dx: 3,
|
|
dy: 2,
|
|
color: ballColor,
|
|
};
|
|
|
|
const stream = canvas.captureStream(15);
|
|
const track = stream.getVideoTracks()[0];
|
|
|
|
let animationId: number;
|
|
|
|
function drawFrame() {
|
|
if (!ctx) return;
|
|
ctx.fillStyle = "#000000";
|
|
ctx.fillRect(0, 0, width, height);
|
|
|
|
ball.x += ball.dx;
|
|
ball.y += ball.dy;
|
|
if (ball.x + ball.radius >= width || ball.x - ball.radius <= 0) ball.dx = -ball.dx;
|
|
if (ball.y + ball.radius >= height || ball.y - ball.radius <= 0) ball.dy = -ball.dy;
|
|
ball.x = Math.max(ball.radius, Math.min(width - ball.radius, ball.x));
|
|
ball.y = Math.max(ball.radius, Math.min(height - ball.radius, ball.y));
|
|
|
|
ctx.beginPath();
|
|
ctx.arc(ball.x, ball.y, ball.radius, 0, Math.PI * 2);
|
|
ctx.fillStyle = ball.color;
|
|
ctx.fill();
|
|
|
|
ctx.fillStyle = "#ffffff";
|
|
ctx.font = "12px Arial";
|
|
ctx.fillText(`Frame: ${Date.now() % 10000}`, 10, 20);
|
|
}
|
|
|
|
function animate() {
|
|
drawFrame();
|
|
animationId = requestAnimationFrame(animate);
|
|
}
|
|
animate();
|
|
|
|
// Store cleanup function on track
|
|
(track as any).stopAnimation = () => {
|
|
if (animationId) cancelAnimationFrame(animationId);
|
|
};
|
|
|
|
track.enabled = true;
|
|
return track;
|
|
};
|
|
|
|
const createSilentAudioTrack = (): MediaStreamTrack => {
|
|
const audioContext = new AudioContext();
|
|
const oscillator = audioContext.createOscillator();
|
|
const gainNode = audioContext.createGain();
|
|
const destination = audioContext.createMediaStreamDestination();
|
|
gainNode.gain.value = 0;
|
|
oscillator.connect(gainNode);
|
|
gainNode.connect(destination);
|
|
oscillator.start();
|
|
const track = destination.stream.getAudioTracks()[0];
|
|
|
|
// Store cleanup function
|
|
(track as any).stopOscillator = () => {
|
|
oscillator.stop();
|
|
audioContext.close();
|
|
};
|
|
|
|
track.enabled = true;
|
|
return track;
|
|
};
|
|
|
|
/* ---------- Peer Types ---------- */
|
|
|
|
interface Peer {
|
|
session_id: string;
|
|
peer_name: string;
|
|
attributes: Record<string, any>;
|
|
muted: boolean;
|
|
video_on: boolean;
|
|
local: boolean;
|
|
dead: boolean;
|
|
connection?: RTCPeerConnection;
|
|
}
|
|
export type { Peer };
|
|
|
|
interface AddPeerConfig {
|
|
peer_id: string;
|
|
peer_name: string;
|
|
should_create_offer?: boolean;
|
|
}
|
|
|
|
interface SessionDescriptionData {
|
|
peer_id: string;
|
|
peer_name: string;
|
|
session_description: RTCSessionDescriptionInit;
|
|
}
|
|
|
|
interface IceCandidateData {
|
|
peer_id: string;
|
|
peer_name: string;
|
|
candidate: RTCIceCandidateInit;
|
|
}
|
|
|
|
interface RemovePeerData {
|
|
peer_name: string;
|
|
peer_id: string;
|
|
}
|
|
|
|
/* ---------- Video Element with Fade-in ---------- */
|
|
|
|
interface VideoProps extends React.VideoHTMLAttributes<HTMLVideoElement> {
|
|
srcObject: MediaStream;
|
|
local?: boolean;
|
|
}
|
|
|
|
const Video: React.FC<VideoProps> = ({ srcObject, local, ...props }) => {
|
|
const refVideo = useRef<HTMLVideoElement>(null);
|
|
const clickHandlerRef = useRef<(() => void) | null>(null);
|
|
const hasUserInteractedRef = useRef<boolean>(false);
|
|
|
|
useEffect(() => {
|
|
if (!refVideo.current || !srcObject) return;
|
|
|
|
const ref = refVideo.current;
|
|
console.log("media-agent - <Video/> - Setting video srcObject:", srcObject);
|
|
|
|
// Remove any existing click handler
|
|
if (clickHandlerRef.current) {
|
|
ref.removeEventListener("click", clickHandlerRef.current);
|
|
clickHandlerRef.current = null;
|
|
ref.style.cursor = "default";
|
|
}
|
|
|
|
// Check if srcObject actually changed
|
|
const oldSrcObject = ref.srcObject;
|
|
ref.srcObject = srcObject;
|
|
|
|
// Always mute initially to satisfy autoplay policy
|
|
ref.muted = true;
|
|
|
|
const playVideo = async () => {
|
|
try {
|
|
// Force restart playback even if it was already playing
|
|
if (!ref.paused) {
|
|
ref.pause();
|
|
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
}
|
|
|
|
await ref.play();
|
|
ref.classList.add("fade-in");
|
|
console.log(`media-agent - <Video/> - Video started playing for ${local ? "local" : "remote"} video`);
|
|
|
|
// For remote videos, only unmute if user has already interacted
|
|
if (!local && hasUserInteractedRef.current && !props.muted) {
|
|
try {
|
|
ref.muted = false;
|
|
} catch (e) {
|
|
console.log("media-agent - <Video/> - Unmute requires user interaction");
|
|
}
|
|
}
|
|
|
|
// If not local and not yet interacted, add click handler for unmuting
|
|
if (!local && !hasUserInteractedRef.current) {
|
|
const handleClick = async () => {
|
|
hasUserInteractedRef.current = true;
|
|
try {
|
|
ref.muted = false;
|
|
ref.removeEventListener("click", handleClick);
|
|
clickHandlerRef.current = null;
|
|
ref.style.cursor = "default";
|
|
console.log(`media-agent - <Video/> - Unmuted after user interaction`);
|
|
} catch (err) {
|
|
console.error(`media-agent - <Video/> - Failed to unmute:`, err);
|
|
}
|
|
};
|
|
|
|
clickHandlerRef.current = handleClick;
|
|
ref.addEventListener("click", handleClick);
|
|
ref.style.cursor = "pointer";
|
|
|
|
// Show visual hint that click is needed for audio
|
|
console.log(`media-agent - <Video/> - Click video to enable audio`);
|
|
}
|
|
} catch (error) {
|
|
console.error(`media-agent - <Video/> - Error playing video for ${local ? "local" : "remote"} video`, error);
|
|
|
|
// If autoplay fails completely, add handler to start playback
|
|
const handleClick = async () => {
|
|
hasUserInteractedRef.current = true;
|
|
try {
|
|
await ref.play();
|
|
ref.muted = false;
|
|
ref.removeEventListener("click", handleClick);
|
|
clickHandlerRef.current = null;
|
|
ref.style.cursor = "default";
|
|
console.log(`media-agent - <Video/> - Video started after user click`);
|
|
} catch (err) {
|
|
console.error(`media-agent - <Video/> - Failed to play after click:`, err);
|
|
}
|
|
};
|
|
|
|
clickHandlerRef.current = handleClick;
|
|
ref.addEventListener("click", handleClick);
|
|
ref.style.cursor = "pointer";
|
|
}
|
|
};
|
|
|
|
// Always try to play when srcObject changes
|
|
const timeoutId = setTimeout(playVideo, 100);
|
|
|
|
return () => {
|
|
clearTimeout(timeoutId);
|
|
if (ref && oldSrcObject !== srcObject) {
|
|
ref.pause();
|
|
if (clickHandlerRef.current) {
|
|
ref.removeEventListener("click", clickHandlerRef.current);
|
|
clickHandlerRef.current = null;
|
|
}
|
|
ref.classList.remove("fade-in");
|
|
ref.style.cursor = "default";
|
|
}
|
|
};
|
|
}, [srcObject, local, props.muted]);
|
|
|
|
// Monitor document interaction separately
|
|
useEffect(() => {
|
|
const handleInteraction = () => {
|
|
hasUserInteractedRef.current = true;
|
|
// Try to unmute any playing remote videos
|
|
if (refVideo.current && !local && !refVideo.current.paused && refVideo.current.muted && !props.muted) {
|
|
try {
|
|
refVideo.current.muted = false;
|
|
console.log("media-agent - <Video/> - Unmuted after document interaction");
|
|
} catch (e) {
|
|
// Still might fail if interaction wasn't sufficient
|
|
}
|
|
}
|
|
};
|
|
|
|
document.addEventListener("click", handleInteraction, { once: true });
|
|
document.addEventListener("keydown", handleInteraction, { once: true });
|
|
|
|
return () => {
|
|
document.removeEventListener("click", handleInteraction);
|
|
document.removeEventListener("keydown", handleInteraction);
|
|
};
|
|
}, [local, props.muted]);
|
|
|
|
return <video ref={refVideo} autoPlay playsInline muted {...props} />;
|
|
};
|
|
|
|
/* ---------- MediaAgent (signaling + peer setup) ---------- */
|
|
|
|
type MediaAgentProps = {
|
|
socketUrl: string;
|
|
session: Session;
|
|
peers: Record<string, Peer>;
|
|
setPeers: React.Dispatch<React.SetStateAction<Record<string, Peer>>>;
|
|
};
|
|
|
|
type JoinStatus = { status: "Not joined" | "Joining" | "Joined" | "Error"; message?: string };
|
|
|
|
const MediaAgent = (props: MediaAgentProps) => {
|
|
const { peers, setPeers, socketUrl, session } = props;
|
|
const [joinStatus, setJoinStatus] = useState<JoinStatus>({ status: "Not joined" });
|
|
const [media, setMedia] = useState<MediaStream | null>(null);
|
|
const [pendingPeers, setPendingPeers] = useState<AddPeerConfig[]>([]);
|
|
|
|
// Refs for singleton resources
|
|
const mediaStreamRef = useRef<MediaStream | null>(null);
|
|
const connectionsRef = useRef<Map<string, RTCPeerConnection>>(new Map());
|
|
const mountedRef = useRef<boolean>(true);
|
|
// Keep track of last processed message to avoid duplicates
|
|
const lastProcessedMessageRef = useRef<string | null>(null);
|
|
const isNegotiatingRef = useRef<Map<string, boolean>>(new Map());
|
|
const makingOfferRef = useRef<Map<string, boolean>>(new Map());
|
|
const initiatedOfferRef = useRef<Set<string>>(new Set());
|
|
const pendingIceCandidatesRef = useRef<Map<string, RTCIceCandidateInit[]>>(new Map());
|
|
|
|
const { sendJsonMessage, lastJsonMessage, readyState } = useWebSocket(socketUrl, {
|
|
share: true,
|
|
onError: (err) => {
|
|
console.error(err);
|
|
},
|
|
onClose: (_event: CloseEvent) => {
|
|
if (!session) return;
|
|
|
|
console.log(`media-agent - ${session.name} Disconnected from signaling server`);
|
|
|
|
// Clean up all peer connections
|
|
connectionsRef.current.forEach((connection, peerId) => {
|
|
connection.close();
|
|
});
|
|
connectionsRef.current.clear();
|
|
|
|
// Mark all peers as dead
|
|
const updatedPeers = { ...peers };
|
|
Object.keys(updatedPeers).forEach((id) => {
|
|
if (!updatedPeers[id].local) {
|
|
updatedPeers[id].dead = true;
|
|
updatedPeers[id].connection = undefined;
|
|
}
|
|
});
|
|
|
|
if (debug) console.log(`media-agent - close`, updatedPeers);
|
|
setPeers(updatedPeers);
|
|
},
|
|
});
|
|
|
|
useEffect(() => {
|
|
for (let peer in peers) {
|
|
console.log(`media-agent - useEffect(peers): Peer ${peers[peer].peer_name}:`, peers[peer]);
|
|
}
|
|
}, [peers]);
|
|
|
|
const addPeer = useCallback(
|
|
async (config: AddPeerConfig) => {
|
|
console.log(`media-agent - addPeer:${config.peer_name} - Callback triggered`, config);
|
|
// Check if connection already exists
|
|
const existingConnection = connectionsRef.current.get(config.peer_id);
|
|
if (existingConnection) {
|
|
const state = existingConnection.connectionState;
|
|
if (state === "new" || state === "connected" || state === "connecting") {
|
|
console.log(
|
|
`media-agent - addPeer:${config.peer_name} Connection already exists for ${config.peer_name} in state: ${state}`
|
|
);
|
|
return;
|
|
} else {
|
|
// Clean up failed/closed connection
|
|
console.log(
|
|
`media-agent - addPeer:${config.peer_name} Cleaning up stale connection for ${config.peer_name} in state: ${state}`
|
|
);
|
|
existingConnection.close();
|
|
connectionsRef.current.delete(config.peer_id);
|
|
}
|
|
}
|
|
|
|
// Queue peer if media not ready
|
|
if (!media) {
|
|
console.log(`media-agent - addPeer:${config.peer_name} - No local media yet, queuing peer`);
|
|
setPendingPeers((prev) => {
|
|
// Avoid duplicate queuing
|
|
if (!prev.some((p) => p.peer_id === config.peer_id)) {
|
|
return [...prev, config];
|
|
}
|
|
return prev;
|
|
});
|
|
return;
|
|
}
|
|
|
|
const peer_id = config.peer_id;
|
|
|
|
// Check if peer already exists and is alive
|
|
if (peer_id in peers && !peers[peer_id].dead) {
|
|
console.log(`media-agent - addPeer:${config.peer_name} - ${config.peer_name} already in peers and alive`);
|
|
return;
|
|
}
|
|
|
|
// Create or revive peer
|
|
const updatedPeers = { ...peers };
|
|
const peer: Peer = {
|
|
session_id: peer_id,
|
|
peer_name: config.peer_name,
|
|
attributes: {},
|
|
muted: updatedPeers[peer_id]?.muted || false,
|
|
video_on: updatedPeers[peer_id]?.video_on !== false,
|
|
local: false,
|
|
dead: false,
|
|
};
|
|
|
|
updatedPeers[peer_id] = peer;
|
|
console.log(
|
|
`media-agent - addPeer:${peer.peer_name} - ${peer_id in peers ? "reviving" : "starting new"} peer ${
|
|
peer.peer_name
|
|
}`
|
|
);
|
|
|
|
// Create RTCPeerConnection
|
|
const connection = new RTCPeerConnection({
|
|
iceServers: [
|
|
{ urls: "stun:stun.l.google.com:19302" },
|
|
{ urls: "stun:stun1.l.google.com:19302" },
|
|
{
|
|
urls: "turns:ketrenos.com:5349",
|
|
username: "ketra",
|
|
credential: "ketran",
|
|
},
|
|
],
|
|
});
|
|
|
|
peer.connection = connection;
|
|
connectionsRef.current.set(peer_id, connection);
|
|
|
|
if (config.should_create_offer) {
|
|
initiatedOfferRef.current.add(peer_id);
|
|
// Pre-set the makingOffer flag to prevent negotiationneeded from firing
|
|
makingOfferRef.current.set(peer_id, true);
|
|
}
|
|
|
|
// Set up event handlers
|
|
connection.addEventListener("negotiationneeded", async (event) => {
|
|
console.log(`media-agent - addPeer:${peer.peer_name} negotiationneeded - ${connection.signalingState}`, event);
|
|
|
|
// Check if we're already making an offer (set by should_create_offer)
|
|
const makingOffer = makingOfferRef.current.get(peer_id) || false;
|
|
if (makingOffer) {
|
|
console.log(`media-agent - addPeer:${peer.peer_name} Already making offer, skipping negotiationneeded...`);
|
|
return;
|
|
}
|
|
|
|
// Check if we're already negotiating
|
|
const isNegotiating = isNegotiatingRef.current.get(peer_id) || false;
|
|
if (isNegotiating) {
|
|
console.log(`media-agent - addPeer:${peer.peer_name} Already negotiating, skipping...`);
|
|
return;
|
|
}
|
|
|
|
// Only proceed if we're in a stable state
|
|
if (connection.signalingState !== "stable") {
|
|
console.log(
|
|
`media-agent - addPeer:${peer.peer_name} Skipping negotiation, signalingState: ${connection.signalingState}`
|
|
);
|
|
return;
|
|
}
|
|
|
|
console.log(`media-agent - addPeer:${peer.peer_name} Handling negotiationneeded for ${peer.peer_name}`);
|
|
|
|
try {
|
|
makingOfferRef.current.set(peer_id, true);
|
|
isNegotiatingRef.current.set(peer_id, true);
|
|
|
|
// Set transceiver direction to avoid m-line ordering issues
|
|
const transceivers = connection.getTransceivers();
|
|
transceivers.forEach((transceiver) => {
|
|
if (transceiver.sender.track) {
|
|
transceiver.direction = "sendrecv";
|
|
}
|
|
});
|
|
|
|
const local_description = await connection.createOffer();
|
|
|
|
// Check if connection is still in the right state
|
|
if (connection.signalingState !== "stable") {
|
|
console.log(`media-agent - addPeer:${peer.peer_name} State changed during offer creation, aborting`);
|
|
return;
|
|
}
|
|
|
|
await connection.setLocalDescription(local_description);
|
|
|
|
sendJsonMessage({
|
|
type: "relaySessionDescription",
|
|
data: { peer_id: peer_id, session_description: local_description },
|
|
});
|
|
console.log(`media-agent - addPeer:${peer.peer_name} Offer sent`, local_description);
|
|
} catch (error) {
|
|
console.error(`media-agent - addPeer:${peer.peer_name} negotiationneeded handling failed:`, error);
|
|
} finally {
|
|
makingOfferRef.current.set(peer_id, false);
|
|
// Keep isNegotiating true until we get an answer
|
|
}
|
|
});
|
|
|
|
connection.addEventListener("connectionstatechange", (event) => {
|
|
console.log(
|
|
`media-agent - addPeer:${peer.peer_name} connectionstatechange - `,
|
|
connection.connectionState,
|
|
event
|
|
);
|
|
if (connection.connectionState === "failed") {
|
|
setTimeout(() => {
|
|
if (connection.connectionState === "failed") {
|
|
console.log(`media-agent - addPeer:${peer.peer_name} Cleaning up failed connection for`, peer.peer_name);
|
|
connectionsRef.current.delete(peer_id);
|
|
// You might want to trigger a reconnection attempt here
|
|
}
|
|
}, 5000);
|
|
}
|
|
});
|
|
|
|
connection.addEventListener("icecandidateerror", (event: Event) => {
|
|
const evt = event as RTCPeerConnectionIceErrorEvent;
|
|
if (evt.errorCode === 701) {
|
|
console.error(`media-agent - addPeer:${peer.peer_name} ICE candidate error for ${peer.peer_name}:`, evt);
|
|
}
|
|
});
|
|
|
|
connection.addEventListener("track", (event) => {
|
|
console.log(
|
|
`media-agent - addPeer:${peer.peer_name} Track event:`,
|
|
event.track.kind,
|
|
"muted:",
|
|
event.track.muted,
|
|
"enabled:",
|
|
event.track.enabled
|
|
);
|
|
|
|
// Force unmute check after a delay
|
|
setTimeout(() => {
|
|
console.log(
|
|
`media-agent - addPeer:${peer.peer_name} Track after delay:`,
|
|
event.track.kind,
|
|
"muted:",
|
|
event.track.muted,
|
|
"enabled:",
|
|
event.track.enabled
|
|
);
|
|
}, 1000);
|
|
});
|
|
|
|
connection.onicecandidate = (event: RTCPeerConnectionIceEvent) => {
|
|
if (!event.candidate) {
|
|
console.log(`media-agent - addPeer:${peer.peer_name} ICE gathering complete: ${connection.connectionState}`);
|
|
return;
|
|
}
|
|
|
|
console.log(`media-agent - addPeer:${peer.peer_name} onicecandidate - `, event.candidate);
|
|
sendJsonMessage({
|
|
type: "relayICECandidate",
|
|
data: {
|
|
peer_id,
|
|
candidate: event.candidate,
|
|
},
|
|
});
|
|
};
|
|
|
|
connection.ontrack = (event: RTCTrackEvent) => {
|
|
console.log(`media-agent - addPeer:${config.peer_name} ontrack event received`, event);
|
|
|
|
let stream: MediaStream;
|
|
if (event.streams && event.streams[0]) {
|
|
stream = event.streams[0];
|
|
} else {
|
|
stream = new MediaStream();
|
|
stream.addTrack(event.track);
|
|
}
|
|
|
|
console.log(`media-agent - addPeer:${config.peer_name} ontrack - Stream:`, stream);
|
|
console.log(`media-agent - addPeer:${config.peer_name} ontrack - Track:`, event.track);
|
|
|
|
// Update peer with stream - use peer_id from closure, not peer object
|
|
setPeers((prevPeers) => {
|
|
const updated = { ...prevPeers };
|
|
if (updated[peer_id]) {
|
|
// Don't check connection reference - just update if peer exists
|
|
updated[peer_id].attributes = {
|
|
...updated[peer_id].attributes,
|
|
srcObject: stream,
|
|
};
|
|
console.log(
|
|
`media-agent - addPeer:${config.peer_name} ontrack - remote ${peer_id} track assigned, srcObject:`,
|
|
stream
|
|
);
|
|
} else {
|
|
console.error(`media-agent - addPeer:${config.peer_name} ontrack - peer ${peer_id} not found in state`);
|
|
}
|
|
return updated;
|
|
});
|
|
};
|
|
|
|
connection.oniceconnectionstatechange = (event) => {
|
|
console.log(
|
|
`media-agent - addPeer:${peer.peer_name} iceconnectionstatechange - `,
|
|
connection.iceConnectionState,
|
|
event
|
|
);
|
|
if (connection.iceConnectionState === "failed") {
|
|
console.log("media-agent - ICE connection failed for", peer.peer_name);
|
|
}
|
|
};
|
|
|
|
// Add local tracks
|
|
console.log(`media-agent - addPeer:${peer.peer_name} Adding local tracks to new peer connection`);
|
|
media.getTracks().forEach((t) => {
|
|
console.log(`media-agent - addPeer:${peer.peer_name} Adding track:`, t.kind, t.enabled);
|
|
connection.addTrack(t, media);
|
|
});
|
|
|
|
// Update peers state
|
|
setPeers(updatedPeers);
|
|
|
|
// Create offer if needed - with proper locking
|
|
if (config.should_create_offer) {
|
|
console.log(`media-agent - addPeer:${peer.peer_name} Creating RTC offer to ${peer.peer_name}`);
|
|
|
|
try {
|
|
isNegotiatingRef.current.set(peer_id, true);
|
|
|
|
const local_description = await connection.createOffer();
|
|
console.log(`media-agent - addPeer:${peer.peer_name} Local offer description created`, local_description);
|
|
|
|
await connection.setLocalDescription(local_description);
|
|
|
|
sendJsonMessage({
|
|
type: "relaySessionDescription",
|
|
data: {
|
|
peer_id,
|
|
session_description: local_description,
|
|
},
|
|
});
|
|
console.log(`media-agent - addPeer:${peer.peer_name} Offer sent to ${peer.peer_name}`);
|
|
} catch (err) {
|
|
console.error(`media-agent - addPeer:${peer.peer_name} Failed to create/send offer:`, err);
|
|
isNegotiatingRef.current.set(peer_id, false);
|
|
} finally {
|
|
// Clear the makingOffer flag after we're done
|
|
makingOfferRef.current.set(peer_id, false);
|
|
}
|
|
}
|
|
},
|
|
[peers, setPeers, media, sendJsonMessage]
|
|
);
|
|
|
|
// Process queued peers when media becomes available
|
|
useEffect(() => {
|
|
if (media && pendingPeers.length > 0) {
|
|
console.log(`media-agent - Processing ${pendingPeers.length} queued peers`);
|
|
const peersToAdd = [...pendingPeers];
|
|
setPendingPeers([]);
|
|
peersToAdd.forEach((config) => addPeer(config));
|
|
}
|
|
}, [media, pendingPeers, addPeer]);
|
|
|
|
const sessionDescription = useCallback(
|
|
async (props: SessionDescriptionData) => {
|
|
const { peer_id, peer_name, session_description } = props;
|
|
const peer = peers[peer_id];
|
|
|
|
console.log(`media-agent - sessionDescription:${peer_name} - Callback triggered.`, props);
|
|
|
|
if (!peer || !peer.connection) {
|
|
console.error(`media-agent - sessionDescription:${peer_name} - No peer for ${peer_id}`);
|
|
return;
|
|
}
|
|
|
|
const pc = peer.connection;
|
|
const desc = new RTCSessionDescription(session_description);
|
|
|
|
// Check for glare condition (both peers creating offers simultaneously)
|
|
const makingOffer = makingOfferRef.current.get(peer_id) || false;
|
|
const offerCollision = desc.type === "offer" && (makingOffer || pc.signalingState !== "stable");
|
|
|
|
// Use polite peer pattern - we're impolite if we initiated the connection
|
|
const weInitiated = initiatedOfferRef.current.has(peer_id);
|
|
const ignoreOffer = weInitiated && offerCollision;
|
|
|
|
if (ignoreOffer) {
|
|
console.log(`media-agent - sessionDescription:${peer_name} - Ignoring offer due to collision`);
|
|
return;
|
|
}
|
|
|
|
console.log(`media-agent - sessionDescription:${peer_name} - `, {
|
|
peer_id,
|
|
type: desc.type,
|
|
signalingState: pc.signalingState,
|
|
});
|
|
|
|
try {
|
|
await pc.setRemoteDescription(desc);
|
|
isNegotiatingRef.current.set(peer_id, false); // Negotiation complete
|
|
console.log(`media-agent - sessionDescription:${peer_name} - Remote description set`);
|
|
|
|
// Process any queued ICE candidates
|
|
const pendingCandidates = pendingIceCandidatesRef.current.get(peer_id);
|
|
if (pendingCandidates && pendingCandidates.length > 0) {
|
|
console.log(
|
|
`media-agent - sessionDescription:${peer_name} - Processing ${pendingCandidates.length} queued ICE candidates`
|
|
);
|
|
|
|
for (const candidate of pendingCandidates) {
|
|
try {
|
|
await pc.addIceCandidate(new RTCIceCandidate(candidate));
|
|
console.log(`media-agent - sessionDescription:${peer_name} - Queued ICE candidate added`);
|
|
} catch (err) {
|
|
console.error(`media-agent - sessionDescription:${peer_name} - Failed to add queued ICE candidate:`, err);
|
|
}
|
|
}
|
|
|
|
// Clear the queue
|
|
pendingIceCandidatesRef.current.delete(peer_id);
|
|
}
|
|
} catch (err) {
|
|
console.error(`media-agent - sessionDescription:${peer_name} - Failed to set remote description:`, err);
|
|
return;
|
|
}
|
|
|
|
// Create answer for offers
|
|
if (session_description.type === "offer") {
|
|
console.log(
|
|
`media-agent - sessionDescription:${peer_name} - Creating answer for received offer in state:`,
|
|
pc.signalingState
|
|
);
|
|
try {
|
|
const local_description = await pc.createAnswer();
|
|
await pc.setLocalDescription(local_description);
|
|
sendJsonMessage({
|
|
type: "relaySessionDescription",
|
|
data: { peer_id, session_description: local_description },
|
|
});
|
|
console.log(`media-agent - sessionDescription:${peer_name} - Answer sent`);
|
|
} catch (err) {
|
|
console.error(`media-agent - sessionDescription:${peer_name} - Failed to create/send answer:`, err);
|
|
}
|
|
}
|
|
},
|
|
[peers, sendJsonMessage]
|
|
);
|
|
|
|
const removePeer = useCallback(
|
|
(props: RemovePeerData) => {
|
|
const { peer_id, peer_name } = props;
|
|
const peer = peers[peer_id];
|
|
if (!peer) {
|
|
console.error(`media-agent - removePeer:${peer_name} - No peer for ${peer_id}`);
|
|
return;
|
|
}
|
|
|
|
console.log(`media-agent - removePeer:${peer_name} - Removing peer.`);
|
|
|
|
// Close and remove connection
|
|
const connection = connectionsRef.current.get(peer_id);
|
|
if (connection) {
|
|
console.log(`media-agent - removePeer:${peer_name} - Closing connection.`);
|
|
connection.close();
|
|
connectionsRef.current.delete(peer_id);
|
|
}
|
|
|
|
// Clean up negotiation state
|
|
isNegotiatingRef.current.delete(peer_id);
|
|
makingOfferRef.current.delete(peer_id);
|
|
initiatedOfferRef.current.delete(peer_id);
|
|
pendingIceCandidatesRef.current.delete(peer_id);
|
|
|
|
// Update peers state
|
|
const updatedPeers = { ...peers };
|
|
if (updatedPeers[peer_id]) {
|
|
updatedPeers[peer_id].dead = true;
|
|
updatedPeers[peer_id].connection = undefined;
|
|
}
|
|
|
|
setPeers(updatedPeers);
|
|
},
|
|
[peers, setPeers]
|
|
);
|
|
|
|
const iceCandidate = useCallback(
|
|
async (props: IceCandidateData) => {
|
|
const { peer_id, peer_name, candidate } = props;
|
|
const peer = peers[peer_id];
|
|
console.log(`media-agent - iceCandidate:${peer_name} - `, { peer_id, candidate, peer });
|
|
|
|
if (!peer?.connection) {
|
|
console.error(`media-agent - iceCandidate:${peer_name} - No peer connection for ${peer_id}`);
|
|
return;
|
|
}
|
|
|
|
// Check if remote description is set
|
|
if (!peer.connection.remoteDescription) {
|
|
console.log(`media-agent - iceCandidate:${peer_name} - Remote description not set yet, queuing ICE candidate`);
|
|
|
|
// Queue the candidate
|
|
const pending = pendingIceCandidatesRef.current.get(peer_id) || [];
|
|
pending.push(candidate);
|
|
pendingIceCandidatesRef.current.set(peer_id, pending);
|
|
return;
|
|
}
|
|
|
|
// Add the ICE candidate
|
|
peer.connection
|
|
.addIceCandidate(new RTCIceCandidate(candidate))
|
|
.then(() => console.log(`media-agent - iceCandidate::${peer_name} - ICE candidate added for ${peer.peer_name}`))
|
|
.catch((err) => console.error(`media-agent - iceCandidate::${peer_name} - Failed to add ICE candidate:`, err));
|
|
},
|
|
[peers]
|
|
);
|
|
|
|
const handleWebSocketMessage = useCallback(
|
|
(data: any) => {
|
|
console.log(`media-agent - WebSocket message received:`, data.type, data.data);
|
|
|
|
switch (data.type) {
|
|
case "join_status":
|
|
setJoinStatus({ status: data.status, message: data.message });
|
|
break;
|
|
case "addPeer":
|
|
addPeer(data.data);
|
|
break;
|
|
case "removePeer":
|
|
removePeer(data.data);
|
|
break;
|
|
case "iceCandidate":
|
|
iceCandidate(data.data);
|
|
break;
|
|
case "sessionDescription":
|
|
sessionDescription(data.data);
|
|
break;
|
|
default:
|
|
break;
|
|
}
|
|
},
|
|
[addPeer, removePeer, iceCandidate, sessionDescription]
|
|
);
|
|
|
|
// Effect runs when new message arrives
|
|
useEffect(() => {
|
|
if (!lastJsonMessage || !session) return;
|
|
|
|
const msgKey = JSON.stringify(lastJsonMessage);
|
|
if (lastProcessedMessageRef.current === msgKey) {
|
|
// already handled this one
|
|
return;
|
|
}
|
|
lastProcessedMessageRef.current = msgKey;
|
|
|
|
handleWebSocketMessage(lastJsonMessage);
|
|
}, [lastJsonMessage, handleWebSocketMessage, session]);
|
|
|
|
// Join lobby when media is ready
|
|
useEffect(() => {
|
|
if (media && joinStatus.status === "Not joined" && readyState === ReadyState.OPEN) {
|
|
console.log(`media-agent - Initiating media join for ${session.name}`);
|
|
setJoinStatus({ status: "Joining" });
|
|
sendJsonMessage({ type: "join" });
|
|
}
|
|
}, [media, joinStatus.status, sendJsonMessage, readyState, session.name]);
|
|
|
|
// Update local peer in peers list
|
|
useEffect(() => {
|
|
if (!session || !media) return;
|
|
|
|
setPeers((prevPeers) => {
|
|
const updated = { ...prevPeers };
|
|
|
|
// Remove old local peer if session ID changed
|
|
Object.keys(updated).forEach((key) => {
|
|
if (updated[key].local && key !== session.id) {
|
|
delete updated[key];
|
|
}
|
|
});
|
|
|
|
// Add or update current local peer
|
|
if (!updated[session.id]) {
|
|
updated[session.id] = {
|
|
peer_name: session.name || "Unknown",
|
|
session_id: session.id,
|
|
local: true,
|
|
muted: true,
|
|
video_on: false,
|
|
attributes: {
|
|
local: true,
|
|
srcObject: media,
|
|
},
|
|
dead: false,
|
|
};
|
|
console.log(`media-agent - Added local peer to peers list`);
|
|
}
|
|
|
|
return updated;
|
|
});
|
|
}, [session, media, setPeers]);
|
|
|
|
// Setup local media
|
|
const setup_local_media = useCallback(async (): Promise<MediaStream> => {
|
|
console.log(`media-agent - Requesting access to local audio/video`);
|
|
const attempt = { get_audio: true, get_video: true };
|
|
let media = null;
|
|
|
|
// Try to get user media with fallback
|
|
while (attempt.get_audio || attempt.get_video) {
|
|
try {
|
|
const constraints: any = {};
|
|
if (attempt.get_audio) constraints.audio = true;
|
|
if (attempt.get_video) constraints.video = true;
|
|
|
|
console.log(`media-agent - Attempting getUserMedia: audio=${attempt.get_audio}, video=${attempt.get_video}`);
|
|
media = await navigator.mediaDevices.getUserMedia(constraints);
|
|
break;
|
|
} catch (error) {
|
|
if (attempt.get_video && attempt.get_audio) {
|
|
console.log(`media-agent - Disabling video, trying audio only`);
|
|
attempt.get_video = false;
|
|
} else if (attempt.get_audio && !attempt.get_video) {
|
|
console.log(`media-agent - Disabling audio, trying video only`);
|
|
attempt.get_video = true;
|
|
attempt.get_audio = false;
|
|
} else {
|
|
console.log(`media-agent - No media available`);
|
|
attempt.get_video = false;
|
|
attempt.get_audio = false;
|
|
}
|
|
}
|
|
}
|
|
|
|
// Build tracks array
|
|
const tracks: MediaStreamTrack[] = [];
|
|
let hasRealAudio = false;
|
|
let hasRealVideo = false;
|
|
|
|
if (media) {
|
|
const audioTracks = media.getAudioTracks();
|
|
const videoTracks = media.getVideoTracks();
|
|
|
|
if (audioTracks.length > 0) {
|
|
tracks.push(audioTracks[0]);
|
|
hasRealAudio = true;
|
|
console.log("media-agent - Using real audio");
|
|
}
|
|
|
|
if (videoTracks.length > 0) {
|
|
videoTracks[0].applyConstraints({
|
|
width: { min: 160, max: 320 },
|
|
height: { min: 120, max: 240 },
|
|
});
|
|
tracks.push(videoTracks[0]);
|
|
hasRealVideo = true;
|
|
console.log("media-agent - Using real video");
|
|
}
|
|
}
|
|
|
|
// Add synthetic tracks if needed
|
|
if (!hasRealAudio) {
|
|
tracks.push(createSilentAudioTrack());
|
|
console.log("media-agent - Using synthetic audio");
|
|
}
|
|
|
|
if (!hasRealVideo) {
|
|
tracks.push(createAnimatedVideoTrack({ name: session.name || "" }));
|
|
console.log("media-agent - Using synthetic video");
|
|
}
|
|
|
|
const finalMedia = new MediaStream(tracks);
|
|
console.log(`media-agent - Media setup complete`);
|
|
return finalMedia;
|
|
}, [session.name]);
|
|
|
|
// Initialize media once
|
|
useEffect(() => {
|
|
mountedRef.current = true;
|
|
|
|
if (mediaStreamRef.current || readyState !== ReadyState.OPEN) return;
|
|
|
|
console.log(`media-agent - Setting up local media`);
|
|
setup_local_media().then((mediaStream) => {
|
|
if (!mountedRef.current) {
|
|
// Component unmounted, clean up
|
|
mediaStream.getTracks().forEach((track) => {
|
|
track.stop();
|
|
if ((track as any).stopAnimation) (track as any).stopAnimation();
|
|
if ((track as any).stopOscillator) (track as any).stopOscillator();
|
|
});
|
|
return;
|
|
}
|
|
|
|
mediaStreamRef.current = mediaStream;
|
|
setMedia(mediaStream);
|
|
});
|
|
|
|
return () => {
|
|
mountedRef.current = false;
|
|
|
|
// Clean up media stream
|
|
if (mediaStreamRef.current) {
|
|
mediaStreamRef.current.getTracks().forEach((track) => {
|
|
track.stop();
|
|
if ((track as any).stopAnimation) (track as any).stopAnimation();
|
|
if ((track as any).stopOscillator) (track as any).stopOscillator();
|
|
});
|
|
mediaStreamRef.current = null;
|
|
}
|
|
|
|
// Clean up connections
|
|
connectionsRef.current.forEach((connection) => connection.close());
|
|
connectionsRef.current.clear();
|
|
};
|
|
}, [readyState, setup_local_media]);
|
|
|
|
return null;
|
|
};
|
|
|
|
/* ---------- MediaControl (UI per peer) ---------- */
|
|
|
|
interface MediaControlProps {
|
|
isSelf: boolean;
|
|
peer: Peer;
|
|
className?: string;
|
|
}
|
|
|
|
const MediaControl: React.FC<MediaControlProps> = ({ isSelf, peer, className }) => {
|
|
const [muted, setMuted] = useState<boolean>(peer?.muted || false);
|
|
const [videoOn, setVideoOn] = useState<boolean>(peer?.video_on !== false);
|
|
const [isValid, setIsValid] = useState<boolean>(false);
|
|
|
|
useEffect(() => {
|
|
if (!peer) return;
|
|
setMuted(peer.muted);
|
|
setVideoOn(peer.video_on);
|
|
}, [peer]);
|
|
|
|
useEffect(() => {
|
|
if (!peer || peer.dead || !peer.attributes?.srcObject) {
|
|
console.log(
|
|
`media-agent - isValid effect:${peer?.peer_name} - No valid media`,
|
|
peer?.dead,
|
|
peer.attributes?.srcObject
|
|
);
|
|
setIsValid(false);
|
|
return;
|
|
}
|
|
|
|
const stream = peer.attributes.srcObject as MediaStream;
|
|
|
|
// For remote peers, check if we have a connected peer connection
|
|
// For local peers, just check if we have live tracks
|
|
if (!peer.local && peer.connection) {
|
|
const isConnected =
|
|
peer.connection.connectionState === "connected" || peer.connection.connectionState === "connecting";
|
|
console.log(
|
|
`media-agent - isValid effect:${peer.peer_name} - Remote peer connection state: ${peer.connection.connectionState}`
|
|
);
|
|
setIsValid(isConnected);
|
|
} else {
|
|
const hasLiveTracks = stream.getTracks().some((track) => track.readyState === "live");
|
|
console.log(`media-agent - isValid effect:${peer.peer_name} - Local peer has live tracks: ${hasLiveTracks}`);
|
|
setIsValid(hasLiveTracks);
|
|
}
|
|
|
|
// Listen for connection state changes for remote peers
|
|
if (!peer.local && peer.connection) {
|
|
const handleConnectionChange = () => {
|
|
const isConnected =
|
|
peer.connection?.connectionState === "connected" || peer.connection?.connectionState === "connecting";
|
|
console.log(
|
|
`media-agent - isValid effect:${peer.peer_name} - Connection state changed: ${peer.connection?.connectionState}`
|
|
);
|
|
setIsValid(isConnected);
|
|
};
|
|
|
|
console.log(`media-agent - isValid effect:${peer.peer_name} - Adding connection state listener`);
|
|
peer.connection.addEventListener("connectionstatechange", handleConnectionChange);
|
|
|
|
return () => {
|
|
peer.connection?.removeEventListener("connectionstatechange", handleConnectionChange);
|
|
};
|
|
}
|
|
|
|
// Listen for track state changes
|
|
const handleTrackStateChange = () => {
|
|
if (!peer.local && peer.connection) {
|
|
const isConnected =
|
|
peer.connection.connectionState === "connected" || peer.connection.connectionState === "connecting";
|
|
console.log(
|
|
`media-agent - isValid effect:${peer.peer_name} - Track state changed, connection state: ${peer.connection.connectionState}`
|
|
);
|
|
setIsValid(isConnected);
|
|
} else {
|
|
const currentlyLive = stream.getTracks().some((track) => track.readyState === "live");
|
|
console.log(
|
|
`media-agent - isValid effect:${peer.peer_name} - Track state changed, live tracks: ${currentlyLive}`
|
|
);
|
|
setIsValid(currentlyLive);
|
|
}
|
|
};
|
|
|
|
stream.getTracks().forEach((track) => {
|
|
track.addEventListener("unmute", handleTrackStateChange);
|
|
track.addEventListener("ended", handleTrackStateChange);
|
|
});
|
|
|
|
return () => {
|
|
stream.getTracks().forEach((track) => {
|
|
track.removeEventListener("unmute", handleTrackStateChange);
|
|
track.removeEventListener("ended", handleTrackStateChange);
|
|
});
|
|
};
|
|
}, [peer, peer.attributes?.srcObject, peer.connection]);
|
|
|
|
useEffect(() => {
|
|
if (!peer || peer.dead || !peer.attributes?.srcObject) return;
|
|
|
|
const stream = peer.attributes.srcObject as MediaStream;
|
|
stream.getAudioTracks().forEach((t) => {
|
|
t.enabled = !muted;
|
|
});
|
|
}, [muted, peer]);
|
|
|
|
useEffect(() => {
|
|
if (!peer || peer.dead || !peer.attributes?.srcObject) return;
|
|
|
|
const stream = peer.attributes.srcObject as MediaStream;
|
|
stream.getVideoTracks().forEach((t) => {
|
|
t.enabled = videoOn;
|
|
});
|
|
}, [videoOn, peer]);
|
|
|
|
const toggleMute = (e: React.MouseEvent | React.TouchEvent) => {
|
|
e.stopPropagation();
|
|
if (peer) {
|
|
peer.muted = !muted;
|
|
setMuted(peer.muted);
|
|
}
|
|
};
|
|
|
|
const toggleVideo = (e: React.MouseEvent | React.TouchEvent) => {
|
|
e.stopPropagation();
|
|
if (peer) {
|
|
peer.video_on = !videoOn;
|
|
setVideoOn(peer.video_on);
|
|
}
|
|
};
|
|
|
|
if (!peer) return null;
|
|
|
|
const colorAudio = isValid ? "primary" : "disabled";
|
|
const colorVideo = isValid ? "primary" : "disabled";
|
|
|
|
return (
|
|
<Box sx={{ display: "flex", flexDirection: "column", alignItems: "center", minWidth: "200px", minHeight: "100px" }}>
|
|
<div className={`MediaControlSpacer ${className}`} />
|
|
<div className={`MediaControl ${className}`} data-peer={peer.session_id}>
|
|
<div className="Controls">
|
|
{isSelf ? (
|
|
<div onTouchStart={toggleMute} onClick={toggleMute}>
|
|
{muted ? <MicOff color={colorAudio} /> : <Mic color={colorAudio} />}
|
|
</div>
|
|
) : (
|
|
<div onTouchStart={toggleMute} onClick={toggleMute}>
|
|
{muted ? <VolumeOff color={colorAudio} /> : <VolumeUp color={colorAudio} />}
|
|
</div>
|
|
)}
|
|
<div onTouchStart={toggleVideo} onClick={toggleVideo}>
|
|
{videoOn ? <Videocam color={colorVideo} /> : <VideocamOff color={colorVideo} />}
|
|
</div>
|
|
</div>
|
|
{isValid ? (
|
|
peer.attributes?.srcObject && (
|
|
<Video
|
|
key={`video-${peer.session_id}-${peer.attributes.srcObject.id}`}
|
|
className="Video"
|
|
data-id={peer.peer_name}
|
|
autoPlay
|
|
srcObject={peer.attributes.srcObject}
|
|
local={peer.local}
|
|
muted={peer.local || muted} // Pass muted state
|
|
/>
|
|
)
|
|
) : (
|
|
<div className="placeholder">Waiting for media…</div>
|
|
)}
|
|
{/* <Moveable
|
|
flushSync={flushSync}
|
|
pinchable
|
|
draggable
|
|
target={document.querySelector(`.MediaControl[data-peer="${peer.session_id}"]`) as HTMLElement}
|
|
resizable
|
|
keepRatio
|
|
hideDefaultLines={false}
|
|
edge
|
|
onDragStart={(e) => e.set(frame.translate)}
|
|
onDrag={(e) => {
|
|
if (Array.isArray(e.beforeTranslate) && e.beforeTranslate.length === 2) {
|
|
frame.translate = [e.beforeTranslate[0], e.beforeTranslate[1]];
|
|
}
|
|
}}
|
|
onResizeStart={(e) => {
|
|
e.setOrigin(["%", "%"]);
|
|
e.dragStart && e.dragStart.set(frame.translate);
|
|
}}
|
|
onResize={(e) => {
|
|
const { translate } = frame;
|
|
e.target.style.width = `${e.width}px`;
|
|
e.target.style.height = `${e.height}px`;
|
|
e.target.style.transform = `translate(${translate[0]}px, ${translate[1]}px)`;
|
|
}}
|
|
onRender={(e) => {
|
|
const { translate } = frame;
|
|
e.target.style.transform = `translate(${translate[0]}px, ${translate[1]}px)`;
|
|
}}
|
|
/> */}
|
|
</div>
|
|
</Box>
|
|
);
|
|
};
|
|
|
|
export { MediaControl, MediaAgent };
|