diff --git a/client/src/MediaControl.tsx b/client/src/MediaControl.tsx index fce6021..6486580 100644 --- a/client/src/MediaControl.tsx +++ b/client/src/MediaControl.tsx @@ -1252,7 +1252,7 @@ const MediaControl: React.FC = ({ isSelf, peer, className }) {isValid ? ( peer.attributes?.srcObject && ( - + ) ) : ( - -
Waiting for media…
- + +
Waiting for media…
+ )} {/* tuple[float, float]: + """ + Calculate dx and dy velocity components based on speed in meters per second. + Frame width represents 1 meter, so pixels per second = width * speed_mps + """ + # Calculate actual time delta since last frame + current_time = time.time() + dt = current_time - self._last_frame_time + self._last_frame_time = current_time + + # Normalize direction vector to ensure consistent speed + dir_x = self.ball["direction_x"] + dir_y = self.ball["direction_y"] + magnitude = np.sqrt(dir_x * dir_x + dir_y * dir_y) + + if magnitude > 0: + dir_x_norm = dir_x / magnitude + dir_y_norm = dir_y / magnitude + else: + dir_x_norm, dir_y_norm = 1.0, 0.0 + + # Convert meters per second to pixels per actual time delta + pixels_per_second = self.width * self.ball["speed_mps"] + pixels_this_frame = pixels_per_second * dt + + # Apply normalized direction to get velocity components + dx = pixels_this_frame * dir_x_norm + dy = pixels_this_frame * dir_y_norm + + return dx, dy async def next_timestamp(self): """Returns (pts, time_base) for 15 FPS video""" @@ -97,16 +151,26 @@ class AnimatedVideoTrack(MediaStreamTrack): # Create black background frame_array = np.zeros((self.height, self.width, 3), dtype=np.uint8) + # Calculate velocity components based on current speed + dx, dy = self._calculate_velocity_components() + # Update ball position ball = self.ball - ball["x"] += ball["dx"] - ball["y"] += ball["dy"] + ball["x"] += dx + ball["y"] += dy - # Bounce off walls + # Bounce off walls and trigger audio events + bounce_occurred = False if ball["x"] + ball["radius"] >= self.width or ball["x"] - ball["radius"] <= 0: - ball["dx"] = -ball["dx"] + ball["direction_x"] = -ball["direction_x"] + bounce_occurred = True if ball["y"] + ball["radius"] >= self.height or ball["y"] - ball["radius"] <= 0: - ball["dy"] = -ball["dy"] + ball["direction_y"] = -ball["direction_y"] + bounce_occurred = True + + # Trigger bounce sound if a bounce occurred + if bounce_occurred and self.audio_track: + self.audio_track.add_bounce_event("bounce") # Keep ball in bounds ball["x"] = max(ball["radius"], min(self.width - ball["radius"], ball["x"])) @@ -121,8 +185,9 @@ class AnimatedVideoTrack(MediaStreamTrack): -1, ) - # Add frame counter text + # Add frame counter and speed text frame_text = f"Frame: {int(time.time() * 1000) % 10000}" + speed_text = f"Speed: {ball['speed_mps']:.2f} m/s" cv2.putText( frame_array, frame_text, @@ -132,6 +197,15 @@ class AnimatedVideoTrack(MediaStreamTrack): (255, 255, 255), 1, ) + cv2.putText( + frame_array, + speed_text, + (10, 40), + cv2.FONT_HERSHEY_SIMPLEX, + 0.5, + (255, 255, 255), + 1, + ) # Convert to VideoFrame frame = VideoFrame.from_ndarray(frame_array, format="bgr24") @@ -142,18 +216,44 @@ class AnimatedVideoTrack(MediaStreamTrack): return frame -class SilentAudioTrack(MediaStreamTrack): +class SyntheticAudioTrack(MediaStreamTrack): """ - Synthetic audio track that generates silence. - Ported from JavaScript createSilentAudioTrack function. + Synthetic audio track that generates audio including bounce sounds. + Originally a silent audio track, now enhanced to generate synthetic audio effects. """ - + kind = "audio" def __init__(self): super().__init__() self.sample_rate = 48000 self.samples_per_frame = 960 # 20ms at 48kHz + self.bounce_queue: list[BounceEvent] = [] # Queue of bounce events to process + self.bounce_duration = 0.1 # 100ms bounce sound duration + self.bounce_amplitude = 0.3 # Amplitude of bounce sound + + def add_bounce_event(self, bounce_type: str = "bounce"): + """Add a bounce event to the audio queue""" + current_time = time.time() + self.bounce_queue.append( + { + "type": bounce_type, + "start_time": current_time, + "end_time": current_time + self.bounce_duration, + } + ) + + def _generate_bounce_sound(self, t: float) -> float: + """Generate a simple bounce sound using a decaying sine wave""" + # Simple bounce sound: combination of two frequencies with decay + freq1 = 800 # Primary frequency + freq2 = 1200 # Secondary frequency + decay = np.exp(-t * 10) # Exponential decay + + sound = ( + np.sin(2 * np.pi * freq1 * t) * 0.7 + np.sin(2 * np.pi * freq2 * t) * 0.3 + ) * decay + return sound * self.bounce_amplitude async def next_timestamp(self): """Returns (pts, time_base) for 20ms audio frames at 48kHz""" @@ -162,15 +262,39 @@ class SilentAudioTrack(MediaStreamTrack): return pts, time_base async def recv(self): - """Generate silent audio frames""" + """Generate audio frames with bounce sounds""" pts, time_base = await self.next_timestamp() + current_time = time.time() - # Create silent audio data in s16 format (required by Opus encoder) - samples = np.zeros((self.samples_per_frame,), dtype=np.int16) + # Create audio data + samples = np.zeros((self.samples_per_frame,), dtype=np.float32) + + # Check for active bounce events and generate sounds + active_bounces: list[BounceEvent] = [] + for bounce in self.bounce_queue: + if current_time < bounce["end_time"]: + # Calculate time within the bounce sound + t = current_time - bounce["start_time"] + if t >= 0: + # Generate bounce sound for this time frame + for i in range(self.samples_per_frame): + sample_time = t + (i / self.sample_rate) + if sample_time <= self.bounce_duration: + samples[i] += self._generate_bounce_sound(sample_time) + active_bounces.append(bounce) + + # Keep only active bounces + self.bounce_queue = active_bounces + + # Clamp samples to prevent distortion + samples = np.clip(samples, -1.0, 1.0) + + # Convert to s16 format (required by Opus encoder) + samples_s16 = (samples * 32767).astype(np.int16) # Convert to AudioFrame frame = AudioFrame.from_ndarray( - samples.reshape(1, -1), format="s16", layout="mono" + samples_s16.reshape(1, -1), format="s16", layout="mono" ) frame.sample_rate = self.sample_rate frame.pts = pts @@ -182,14 +306,21 @@ class SilentAudioTrack(MediaStreamTrack): def create_synthetic_tracks(session_name: str) -> dict[str, MediaStreamTrack]: """ Create synthetic audio and video tracks for WebRTC streaming. - + Args: session_name: Name to use for generating video track colors - + Returns: Dictionary containing 'video' and 'audio' tracks + + Note: + To change ball speed, use: tracks["video"].set_ball_speed(speed_in_mps) + where speed_in_mps is meters per second (frame width = 1 meter) """ - return { - "video": AnimatedVideoTrack(name=session_name), - "audio": SilentAudioTrack() - } + # Create audio track first + audio_track = SyntheticAudioTrack() + + # Create video track with reference to audio track for bounce events + video_track = AnimatedVideoTrack(name=session_name, audio_track=audio_track) + + return {"video": video_track, "audio": audio_track}