commit 642935764c04d63f24c71d309e871b217818cab3 Author: James Ketrenos Date: Sat Aug 23 20:32:41 2025 -0700 Initial commit diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..1308843 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,28 @@ +node_modules +build +dist +__pycache__ +*.pyc +*.pyo +*.pyd +*.log +*.swp +*.swo +.DS_Store +.vscode +.idea +*.sublime-workspace +*.sublime-project +.env +.env.* +dev-keys +*.pem +*.key +coverage +*.bak +*.tmp +*.local +package-lock.json +yarn.lock +pnpm-lock.yaml +*docker-compose.override.yml diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a312226 --- /dev/null +++ b/.gitignore @@ -0,0 +1,59 @@ +# Node +node_modules/ +build/ +dist/ +.env +.env.* +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* + +# Editor +.vscode/ +.idea/ +*.swp +*.swo +.DS_Store +*.sublime-workspace +*.sublime-project + +# Python +__pycache__/ +*.pyc +*.pyo +*.pyd +.env +.venv +.env/ +.venv/ + +# Certificates and keys +dev-keys/ +*.pem +*.key + +# OS +Thumbs.db +Desktop.ini + +# Docker +*.pid + +# Misc +*.bak +*.tmp + +# Test coverage +coverage/ + +# Local config +*.local + +# Ignore lock files +package-lock.json +yarn.lock +pnpm-lock.yaml + +# Ignore docker-compose override +*docker-compose.override.yml diff --git a/Dockerfile.frontend b/Dockerfile.frontend new file mode 100644 index 0000000..9585fa6 --- /dev/null +++ b/Dockerfile.frontend @@ -0,0 +1,41 @@ +FROM ubuntu:noble + +RUN apt-get -q update \ + && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ + ca-certificates curl gnupg \ + curl \ + nano \ + sqlite3 \ + psmisc \ + wget \ + jq \ + less \ + git \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/{apt,dpkg,cache,log} +RUN mkdir -p /etc/apt/keyrings +RUN curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg + +# https://nodejs.org/en/about/previous-releases +ENV NODE_MAJOR=24 +RUN echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list + +RUN apt-get -q update \ + && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ + nodejs \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/{apt,dpkg,cache,log} + +COPY /client /client +WORKDIR /client + +# Set environment variable for production mode (default: development) +ENV PRODUCTION=false + +# Disable HTTPS by default for npm development server +ENV HTTPS=false + +COPY ./client/entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh + +ENTRYPOINT [ "/entrypoint.sh" ] diff --git a/Dockerfile.server b/Dockerfile.server new file mode 100644 index 0000000..ae0838c --- /dev/null +++ b/Dockerfile.server @@ -0,0 +1,50 @@ +FROM ubuntu:oracular + +# Install some utilities frequently used +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y \ + curl \ + gpg \ + iputils-ping \ + jq \ + nano \ + rsync \ + wget \ + python3 \ + python3-pip \ + # python3-venv \ + # python3-dev \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/{apt,dpkg,cache,log} + +# Install latest Python3 +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/{apt,dpkg,cache,log} + + +# Install uv using the official Astral script +RUN curl -Ls https://astral.sh/uv/install.sh | bash +ENV PATH=/root/.local/bin:$PATH + +WORKDIR /server + +# Copy code + + +# Copy code and entrypoint +COPY ./server /server +COPY ./client/build /client/build +COPY ./server/entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh + +# Set environment variable for production mode (default: development) +ENV PRODUCTION=false + +# the cache and target directories are on different filesystems, hardlinking may not be supported. +ENV UV_LINK_MODE=copy + +EXPOSE 8000 + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..b43dad1 --- /dev/null +++ b/README.md @@ -0,0 +1,125 @@ +# AI Voicebot + +AI Voicebot is an agentic AI agent that communicates via ICE and TURN running on a coturn server. + +coturn provides ICE and related specs: + +* RFC 5245 - ICE +* RFC 5768 – ICE–SIP +* RFC 6336 – ICE–IANA Registry +* RFC 6544 – ICE–TCP +* RFC 5928 - TURN Resolution Mechanism + +## To use + +Set the environment variable COTURN_SERVER to point to the URL running the +coturn server by modifying the .env file: + +```.env +COTURN_SERVER="turns:ketrenos.com:5349" +``` + +You then launch the application, providing + +## Architecture + +The system is broken into two major components: client and server + +### client + +The frontend client is written using React, exposed via a static build of the +client through the server's static file endpoint. + +Implementation of the client is in the `client` subdirectory. + +Provides a Web UI for starting a human chat session. A lobby is created based on the URL, and any user with that URL can join that lobby. + +The client uses RTCPeerConnection, RTCSessionDescription, RTCIceCandidate, MediaStream, navigator.getUserMedia, navigator.mediaDevices, and associated APIs for creating audio (via audio tag) and video (via video tag) media instantiations in the Web UI client. + +The client also exposes the ability to add new AI "users" to the lobby. When creating a user, you can provide a brief description of the user. The server +will use that description to generate an AI person, including profile picture, voice signature used for text-to-speech, etc. + +### server + +The backend server is written in Python and the OpenAI Agentic AI SDK, connecting to an OPENAI compatible server running at OPENAI_BASE_URL. + +Implementation of the client is in the `server` subdirectory. + +The model used by the server for LLM communication is set via OPENAI_MODEL. For example: + +```.env +OPENAI_BASE_URL=http://192.168.1.198:8000/v3 +OPENAI_MODEL=Qwen/Qwen3-8B +OPENAI_API_KEY=sk-XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX +``` + + +If you want to use OpenAI instead of a self hosted service, do not set OPENAI_BASE_URL and set the OPENAI_API_KEY accordingly. + +The server provides the AI chatbot and hosts the static files for the client frontend. + + +### Speech-to-Text and Text-to-Speech Configuration + +The server supports pluggable speech-to-text (STT) and text-to-speech (TTS) backends. To configure these, set the following environment variables in your `.env` file: + +``` +STT_MODEL=your-speech-to-text-model +TTS_MODEL=your-text-to-speech-model +``` + +These models are used to transcribe incoming audio and synthesize AI responses, respectively. (See future roadmap for planned model support.) + + +The server communicates with the coturn server in the same manner as the client, only via Python instead. + + +The server exposes an http endpoint via FastAPI. This endpoint exposes the following capabilities: + +1. Lobby creation +2. User management within lobby +3. AI agent creation for a lobby +4. Connection details for the voice system to attach / detach to audio coturn streams as users join / leave. + + +Once an AI agent is added to a lobby, it joins the audio stream(s) for that lobby. + + +Audio input is then passed to the speech-to-text processor to provide a stream of text with time markers. + + +That text is then passed to the language processing layer of the AI agent, which passes it to the LLM for a response. + + +The response is then passed through the text-to-speech processor, with the output stream being routed back to coturn server for dispatch to the human UI viewers. + +## Lobby Features + +- **Player Management:** Players can join/leave lobbies, and their status is tracked in real time. +- **AI and Human Users:** Both AI and human users can participate in lobbies. AI users are generated with custom profiles and voices. + +### Media and Peer Connection Handling + +- **WebRTC Integration:** The client uses WebRTC APIs (RTCPeerConnection, RTCSessionDescription, RTCIceCandidate, MediaStream, etc.) to manage real-time audio/video streams between users and AI agents. +- **Dynamic Peer Management:** Peers are dynamically added/removed as users join or leave lobbies. The system handles ICE candidate negotiation, connection state changes, and media stream routing. +- **Audio/Video UI:** Audio and video streams are rendered in the browser using standard HTML media elements. + +### Extensibility and Planned Enhancements + +- **Pluggable STT/TTS Backends:** Support for additional speech-to-text and text-to-speech providers is planned. +- **Custom AI Agent Personalities:** Future versions will allow more detailed customization of AI agent behavior, voice, and appearance. +- **Improved Moderation and Controls:** Features for lobby moderation, user muting, and reporting are under consideration. +- **Mobile and Accessibility Improvements:** Enhanced support for mobile devices and accessibility features is on the roadmap. + +--- + +## Roadmap + +- [ ] Add support for multiple STT/TTS providers +- [ ] Expand game logic and add new game types +- [ ] Improve AI agent customization options +- [ ] Add lobby moderation and user controls +- [ ] Enhance mobile and accessibility support + +Contributions and feature requests are welcome! + diff --git a/client/.babelrc b/client/.babelrc new file mode 100644 index 0000000..6e867f9 --- /dev/null +++ b/client/.babelrc @@ -0,0 +1,4 @@ +{ + "presets": [ "@babel/env", "@babel/preset-react" ], + "plugins": [ "@babel/plugin-proposal-class-properties" ] +} diff --git a/client/README.md b/client/README.md new file mode 100644 index 0000000..16fe669 --- /dev/null +++ b/client/README.md @@ -0,0 +1,7 @@ +To deploy: + +```bash +export PUBLIC_URL=/ai-voicebot +npm run build +rsync --delete -avrpl build/ webserver:/var/www/ketrenos.com/ai-voicebot/ +``` diff --git a/client/entrypoint.sh b/client/entrypoint.sh new file mode 100644 index 0000000..40023cc --- /dev/null +++ b/client/entrypoint.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +# Launch server in production or development mode +if [ "$PRODUCTION" = "true" ]; then + export REACT_APP_AI_VOICECHAT_BUILD="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + npm install + npm run build +else + export REACT_APP_AI_VOICECHAT_BUILD="Development" + npm install + if [ -f "${SSL_CERTFILE}" ] && [ -f "${SSL_KEYFILE}" ]; then + export HTTPS=true + else + echo "SSL files not found, starting frontend WS without SSL." + export HTTPS=false + fi + npm start +fi \ No newline at end of file diff --git a/client/favicon.xcf b/client/favicon.xcf new file mode 100644 index 0000000..d4b65f7 Binary files /dev/null and b/client/favicon.xcf differ diff --git a/client/package.json b/client/package.json new file mode 100644 index 0000000..d3c5cfc --- /dev/null +++ b/client/package.json @@ -0,0 +1,55 @@ +{ + "name": "ai-voicebot", + "version": "0.1.0", + "private": true, + "dependencies": { + "@emotion/react": "^11.14.0", + "@emotion/styled": "^11.14.1", + "@mui/icons-material": "^7.3.1", + "@mui/material": "^7.3.1", + "@mui/styles": "^6.5.0", + "@mui/utils": "^7.3.1", + "fast-deep-equal": "^3.1.3", + "http-proxy-middleware": "^3.0.5", + "react": "^19.1.1", + "react-dom": "^19.1.1", + "react-movable": "^3.4.1", + "react-moveable": "^0.56.0", + "react-router-dom": "^7.8.2", + "react-scripts": "5.0.1", + "socket.io-client": "^4.8.1", + "web-vitals": "^5.1.0" + }, + "devDependencies": { + "typescript": "^5.4.5", + "@types/node": "^20.11.30", + "@types/react": "^18.2.70", + "@types/react-dom": "^18.2.19", + "@types/react-router-dom": "^5.3.3" + }, + "scripts": { + "start": "react-scripts start", + "build": "react-scripts build", + "test": "react-scripts test", + "eject": "react-scripts eject", + "type-check": "tsc --noEmit" + }, + "eslintConfig": { + "extends": [ + "react-app", + "react-app/jest" + ] + }, + "browserslist": { + "production": [ + ">0.2%", + "not dead", + "not op_mini all" + ], + "development": [ + "last 1 chrome version", + "last 1 firefox version", + "last 1 safari version" + ] + } +} diff --git a/client/public/favicon.ico b/client/public/favicon.ico new file mode 100644 index 0000000..9c9941f Binary files /dev/null and b/client/public/favicon.ico differ diff --git a/client/public/index.html b/client/public/index.html new file mode 100755 index 0000000..8609997 --- /dev/null +++ b/client/public/index.html @@ -0,0 +1,45 @@ + + + + + + + + + + + + + + + AI Voice Chat + + + +
+ + + diff --git a/client/public/logo192.png b/client/public/logo192.png new file mode 100644 index 0000000..dec0696 Binary files /dev/null and b/client/public/logo192.png differ diff --git a/client/public/logo512.png b/client/public/logo512.png new file mode 100644 index 0000000..941a267 Binary files /dev/null and b/client/public/logo512.png differ diff --git a/client/public/manifest.json b/client/public/manifest.json new file mode 100644 index 0000000..3d9c511 --- /dev/null +++ b/client/public/manifest.json @@ -0,0 +1,25 @@ +{ + "short_name": "AI Voicebot", + "name": "AI Voicebot", + "icons": [ + { + "src": "favicon.ico", + "sizes": "64x64 32x32 24x24 16x16", + "type": "image/x-icon" + }, + { + "src": "logo192.png", + "type": "image/png", + "sizes": "192x192" + }, + { + "src": "logo512.png", + "type": "image/png", + "sizes": "512x512" + } + ], + "start_url": ".", + "display": "standalone", + "theme_color": "#000000", + "background_color": "#ffffff" +} diff --git a/client/public/robots.txt b/client/public/robots.txt new file mode 100644 index 0000000..e9e57dc --- /dev/null +++ b/client/public/robots.txt @@ -0,0 +1,3 @@ +# https://www.robotstxt.org/robotstxt.html +User-agent: * +Disallow: diff --git a/client/src/App.css b/client/src/App.css new file mode 100755 index 0000000..de25b97 --- /dev/null +++ b/client/src/App.css @@ -0,0 +1,229 @@ +body { + font-family: 'Droid Sans', 'Arial Narrow', Arial, sans-serif; + overflow: hidden; +} + +#root { + width: 100vw; +/* height: 100vh; breaks on mobile -- not needed */ +} + +.Table { + display: flex; + position: absolute; + top: 0; + left: 0; + width: 100%; + bottom: 0; + flex-direction: row; + /* background-image: url("./assets/tabletop.png"); */ +} + +.Table .Dialogs { + z-index: 10000; + display: flex; + justify-content: space-around; + align-items: center; + position: absolute; + top: 0; + left: 0; + bottom: 0; + right: 0; +} + +.Table .Dialogs .Dialog { + display: flex; + position: absolute; + flex-shrink: 1; + flex-direction: column; + padding: 0.25rem; + left: 0; + right: 0; + top: 0; + bottom: 0; + justify-content: space-around; + align-items: center; + z-index: 60000; +} + +.Table .Dialogs .Dialog > div { + display: flex; + padding: 1rem; + flex-direction: column; +} + +.Table .Dialogs .Dialog > div > div:first-child { + padding: 1rem; +} + +.Table .Dialogs .TurnNoticeDialog { + background-color: #7a680060; +} + +.Table .Dialogs .ErrorDialog { + background-color: #40000060; +} + +.Table .Dialogs .WarningDialog { + background-color: #00000060; +} + +.Table .Game { + position: relative; + display: flex; + flex-direction: column; + flex-grow: 1; +} + +.Table .Board { + display: flex; + position: relative; + flex-grow: 1; + z-index: 500; +} + +.Table .PlayersStatus { + z-index: 500; /* Under Hand */ +} + +.Table .PlayersStatus.ActivePlayer { + z-index: 1500; /* On top of Hand */ +} + +.Table .Hand { + display: flex; + position: relative; + height: 11rem; + z-index: 10000; +} + +.Table .Sidebar { + display: flex; + flex-direction: column; + justify-content: space-between; + width: 25rem; + max-width: 25rem; + overflow: hidden; + z-index: 5000; +} + +.Table .Sidebar .Chat { + display: flex; + position: relative; + flex-grow: 1; +} + +.Table .Trade { + display: flex; + position: relative; + z-index: 25000; + align-self: center; +} + +.Table .Dialogs { + position: absolute; + display: flex; + top: 0; + bottom: 0; + right: 0; + left: 0; + justify-content: space-around; + align-items: center; + z-index: 20000; + pointer-events: none; +} + +.Table .Dialogs > * { + pointer-events: all; +} + +.Table .ViewCard { + display: flex; + position: absolute; + top: 0; + left: 0; + right: 0; + bottom: 0; +} + +.Table .Winner { + display: flex; + position: absolute; + top: 0; + left: 0; + right: 0; + bottom: 0; +} + + +.Table .HouseRules { + display: flex; + position: absolute; + top: 0; + left: 0; + right: 0; + bottom: 0; +} + +.Table .ChooseCard { + display: flex; + position: relative; + top: 0; + left: 0; + right: 0; + bottom: 0; +} + +.Table button { + margin: 0.25rem; + background-color: white; + border: 1px solid black; /* why !important */ +} + +.Table .MuiButton-text { + padding: 0.25rem 0.55rem; +} + +.Table button:disabled { + opacity: 0.5; + border: 1px solid #ccc; /* why !important */ +} + +.Table .ActivitiesBox { + display: flex; + flex-direction: column; + position: absolute; + left: 1em; + top: 1em; +} + +.Table .DiceRoll { + display: flex; + flex-direction: column; + position: relative; + /* + left: 1rem; + top: 5rem;*/ + flex-wrap: wrap; + justify-content: left; + align-items: left; + z-index: 1000; +} + +.Table .DiceRoll div:not(:last-child) { + border: 1px solid black; + background-color: white; + padding: 0.25rem 0.5rem; + border-radius: 0.25rem; +} +.Table .DiceRoll div:last-child { + display: flex; + flex-direction: row; +} + +.Table .DiceRoll .Dice { + margin: 0.25rem; + width: 2.75rem; + height: 2.75rem; + border-radius: 0.5rem; +} \ No newline at end of file diff --git a/client/src/App.tsx b/client/src/App.tsx new file mode 100644 index 0000000..c1be5a1 --- /dev/null +++ b/client/src/App.tsx @@ -0,0 +1,218 @@ +import React, { useState, useEffect, KeyboardEvent, useRef } from "react"; +import { Input, Paper, Typography } from "@mui/material"; + +import { GlobalContext, GlobalContextType } from "./GlobalContext"; +import { UserList } from "./UserList"; +import "./App.css"; +import { base } from "./Common"; +import { Box, Button } from "@mui/material"; +import { BrowserRouter as Router, Route, Routes, useParams } from "react-router-dom"; + +console.log(`AI Voice Chat Build: ${process.env.REACT_APP_AI_VOICECHAT_BUILD}`); + +type LobbyProps = { + lobbyId: string; + sessionId: string; +}; +const Lobby: React.FC = (props: LobbyProps) => { + const { lobbyId, sessionId } = props; + const [editName, setEditName] = useState(""); + const [name, setName] = useState(""); + const [ws, setWs] = useState(undefined); + const [error, setError] = useState(null); + const [global, setGlobal] = useState({ + connected: false, + ws: undefined, + name: "", + chat: [], + }); + + useEffect(() => { + console.log(global); + }, [global]); + + const onWsMessage = (event: MessageEvent) => { + const data = JSON.parse(event.data); + switch (data.type) { + case "error": + setError(data.error); + break; + default: + break; + } + }; + const refWsMessage = useRef(onWsMessage); + + useEffect(() => { + refWsMessage.current = onWsMessage; + }); + + useEffect(() => { + if (!ws) { + return; + } + const cbMessage = (e: MessageEvent) => refWsMessage.current(e); + ws.addEventListener("message", cbMessage); + return () => { + ws.removeEventListener("message", cbMessage); + }; + }, [ws, refWsMessage]); + + // Setup websocket connection on mount (only once) + useEffect(() => { + if (!lobbyId) { + console.log("No lobby ID"); + return; + } + let loc = window.location, + new_uri; + if (loc.protocol === "https:") { + new_uri = "wss"; + } else { + new_uri = "ws"; + } + new_uri = `${new_uri}://${loc.host}${base}/ws/lobby/${lobbyId}`; + const socket = new WebSocket(new_uri); + socket.onopen = () => { + console.log("WebSocket connected"); + setGlobal((g: GlobalContextType) => ({ ...g, connected: true })); + if (name) { + socket.send(JSON.stringify({ type: "set_name", name })); + } + }; + setWs(socket); + setGlobal((g: GlobalContextType) => ({ ...g, ws: socket })); + return () => { + setGlobal((g: GlobalContextType) => ({ ...g, connected: false, ws: undefined })); + if (socket.readyState !== 0) { + socket.close(); + } + }; + // Only run once on mount + // eslint-disable-next-line + }, []); + + // Update global context and send set_name when name changes + useEffect(() => { + if (!ws || !global.connected || global.name === name) { + return; + } + setGlobal((g: GlobalContextType) => ({ ...g, name })); + console.log("Sending set_name", name); + ws.send(JSON.stringify({ type: "set_name", name })); + }, [name, ws, global]); + + const handleKeyDown = (event: KeyboardEvent): void => { + if (event.key === "Enter") { + event.preventDefault(); + if (!editName.trim()) { + return; + } + setName(editName.trim()); + setEditName(""); + } + }; + + return ( + + {!global.connected ? ( +

Connecting to server...

+ ) : ( + + {global.name && } + + {!global.name && ( + + Enter your name to join: + + { + setEditName(e.target.value); + }} + onKeyDown={handleKeyDown} + placeholder="Your name" + /> + + + + )} + + )} + {error && ( + + {error} + + )} +
+ ); +}; + +const App = () => { + const [sessionId, setSessionId] = useState(undefined); + const [error, setError] = useState(null); + const { lobbyId = "default" } = useParams<{ lobbyId: string }>(); + + useEffect(() => { + console.log(`App - sessionId`, sessionId); + }, [sessionId]); + + useEffect(() => { + if (sessionId) { + return; + } + fetch(`${base}/api/lobby`, { + method: "GET", + cache: "no-cache", + credentials: "same-origin", + headers: { + "Content-Type": "application/json", + }, + }) + .then((res) => { + if (res.status >= 400) { + const error = `Unable to connect to AI Voice Chat server! Try refreshing your browser in a few seconds.`; + console.error(error); + setError(error); + } + return res.json(); + }) + .then((data) => { + setSessionId(data.session); + }) + .catch((error) => {}); + }, [sessionId, setSessionId]); + + return ( + + {!sessionId &&

Connecting to server...

} + {sessionId && ( + + + } path={`${base}/:lobbyId`} /> + } path={`${base}`} /> + + + )} + {error && ( + + {error} + + )} +
+ ); +}; + +export default App; diff --git a/client/src/Common.ts b/client/src/Common.ts new file mode 100644 index 0000000..1ce4345 --- /dev/null +++ b/client/src/Common.ts @@ -0,0 +1,18 @@ +function debounce void>(fn: T, ms: number) { + let timer: ReturnType; + return function(this: any, ...args: Parameters) { + clearTimeout(timer); + timer = setTimeout(() => { + timer = null as any; + fn.apply(this, args); + }, ms); + }; +} + +const base = process.env.PUBLIC_URL || ""; + +const assetsPath = `${base}/assets`; +const gamesPath = `${base}`; + +export { base, debounce, assetsPath, gamesPath }; +export {}; diff --git a/client/src/GlobalContext.tsx b/client/src/GlobalContext.tsx new file mode 100644 index 0000000..cbcb273 --- /dev/null +++ b/client/src/GlobalContext.tsx @@ -0,0 +1,19 @@ +import { createContext } from "react"; + +interface GlobalContextType { + connected: boolean; + ws?: WebSocket; + name?: string; + chat?: any[]; + [key: string]: any; +} + +const GlobalContext = createContext({ + connected: false, + ws: undefined, + name: "", + chat: [] +}); + +export { GlobalContext }; +export type { GlobalContextType }; diff --git a/client/src/LobbyMessage.ts b/client/src/LobbyMessage.ts new file mode 100644 index 0000000..5d18778 --- /dev/null +++ b/client/src/LobbyMessage.ts @@ -0,0 +1 @@ +type LobbyMessage = {} \ No newline at end of file diff --git a/client/src/MediaControl.css b/client/src/MediaControl.css new file mode 100644 index 0000000..8ee364a --- /dev/null +++ b/client/src/MediaControl.css @@ -0,0 +1,92 @@ +.MediaControlSpacer { + display: flex; + width: 5rem; + min-width: 5rem; + height: 3.75rem; + min-height: 3.75rem; + background-color: #444; + border-radius: 0.25rem; +} + +.MediaControlSpacer.Medium { + width: 11.5em; + height: 8.625em; + min-width: 11.5em; + min-height: 8.625em; +} + + +.MediaControl { + display: flex; + position: fixed; + flex-direction: row; + justify-content: flex-end; + align-items: center; + width: 5rem; + height: 3.75rem; + min-width: 5rem; + min-height: 3.75rem; + z-index: 50000; +} + +.MediaControl .Video { + position: relative; + width: 100%; + height: 100%; + background-color: #444; + border-radius: 0.25rem; + border: 1px solid black; +} + +.MediaControl.Medium { + width: 11.5em; + height: 8.625em; + min-width: 11.5em; + min-height: 8.625em; +} + +.MediaControl > div { + display: flex; + position: absolute; + top: 0; + left: 0; + display: flex; + flex-direction: column; + align-items: center; + margin-right: 0.25rem; +} + +.MediaControl .Controls { + display: flex; + position: absolute; + left: 0.5em; + bottom: 0.5em; + justify-content: flex-end; + z-index: 1; +} + +.MediaControl.Small .Controls { + left: 0; + bottom: unset; + justify-content: center; +} + +.MediaControl .Controls > div { + display: flex; + border-radius: 0.25em; + cursor: pointer; + padding: 0.25em; +} + +.MediaControl .Controls > div:hover { + background-color: #d0d0d0; +} + +.moveable-control-box { + border: none; + --moveable-color: unset !important; +} + +.moveable-control-box .moveable-direction { + border: none !important; +} \ No newline at end of file diff --git a/client/src/MediaControl.tsx b/client/src/MediaControl.tsx new file mode 100644 index 0000000..8044126 --- /dev/null +++ b/client/src/MediaControl.tsx @@ -0,0 +1,728 @@ +import React, { useState, useEffect, useRef, useCallback, useContext } from "react"; +import Moveable from "react-moveable"; +import "./MediaControl.css"; +import VolumeOff from "@mui/icons-material/VolumeOff"; +import VolumeUp from "@mui/icons-material/VolumeUp"; +import MicOff from "@mui/icons-material/MicOff"; +import Mic from "@mui/icons-material/Mic"; +import VideocamOff from "@mui/icons-material/VideocamOff"; +import Videocam from "@mui/icons-material/Videocam"; +import { GlobalContext } from "./GlobalContext"; +import Box from "@mui/material/Box"; + +const debug = true; + +// Types for peer and track context +interface Peer { + name: string; + hasAudio: boolean; + hasVideo: boolean; + attributes: Record; + muted: boolean; + videoOn: boolean; + local: boolean; + dead: boolean; + connection?: RTCPeerConnection; +} + +interface TrackContext { + media: MediaStream; + audio: boolean; + video: boolean; +} + +interface AddPeerConfig { + peer_id: string; + hasAudio: boolean; + hasVideo: boolean; + should_create_offer?: boolean; +} + +interface SessionDescriptionData { + peer_id: string; + session_description: RTCSessionDescriptionInit; +} + +interface IceCandidateData { + peer_id: string; + candidate: RTCIceCandidateInit; +} + +interface RemovePeerData { + peer_id: string; +} + +interface VideoProps extends React.VideoHTMLAttributes { + srcObject: MediaProvider; + local?: boolean; +} + +const Video: React.FC = ({ srcObject, local, ...props }) => { + const refVideo = useRef(null); + useEffect(() => { + if (!refVideo.current) { + return; + } + const ref = refVideo.current; + if (debug) console.log("media-control - video