diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index a8056edd4d..fc7ca32364 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -3,17 +3,12 @@ name: "publish" # change this when ready to release if you want CI/CD on: workflow_dispatch: - inputs: - interactionId: - description: "Discord Interaction ID" - required: false - type: string env: - CN_APPLICATION: cap/cap + CN_APPLICATION: inflight/inflight APP_CARGO_TOML: apps/desktop/src-tauri/Cargo.toml - SENTRY_ORG: cap-s2 - SENTRY_PROJECT: cap-desktop + SENTRY_ORG: inflight-software + SENTRY_PROJECT: inflight-desktop jobs: draft: @@ -97,36 +92,6 @@ jobs: draft: true generate_release_notes: true - - name: Update Discord interaction - if: ${{ inputs.interactionId != '' }} - uses: actions/github-script@v7 - with: - script: | - async function main() { - const token = await core.getIDToken("cap-discord-bot"); - const cnReleaseId = JSON.parse(`${{ steps.create_cn_release.outputs.stdout }}`).id; - - const resp = await fetch("https://cap-discord-bot.brendonovich.workers.dev/github-workflow", { - method: "POST", - body: JSON.stringify({ - type: "release-ready", - tag: "${{ steps.create_tag.outputs.tag_name }}", - version: "${{ steps.read_version.outputs.value }}", - releaseUrl: "${{ steps.create_gh_release.outputs.url }}", - interactionId: "${{ inputs.interactionId }}", - cnReleaseId - }), - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${token}`, - } - }); - - if(resp.status !== 200) throw new Error(await resp.text()); - } - - main(); - build: needs: draft if: ${{ needs.draft.outputs.needs_release == 'true' }} @@ -138,9 +103,9 @@ jobs: matrix: settings: - target: x86_64-apple-darwin - runner: macos-latest-xlarge + runner: macos-latest - target: aarch64-apple-darwin - runner: macos-latest-xlarge + runner: macos-latest - target: x86_64-pc-windows-msvc runner: windows-latest env: @@ -155,13 +120,13 @@ jobs: run: echo "${{ secrets.APPLE_API_KEY_FILE }}" > api.p8 - uses: apple-actions/import-codesign-certs@v2 - if: ${{ matrix.settings.runner == 'macos-latest-xlarge' }} + if: ${{ startsWith(matrix.settings.runner, 'macos') }} with: p12-file-base64: ${{ secrets.APPLE_CERTIFICATE }} p12-password: ${{ secrets.APPLE_CERTIFICATE_PASSWORD }} - name: Verify certificate - if: ${{ matrix.settings.runner == 'macos-latest-xlarge' }} + if: ${{ startsWith(matrix.settings.runner, 'macos') }} run: security find-identity -v -p codesigning ${{ runner.temp }}/build.keychain - name: Rust setup @@ -280,39 +245,3 @@ jobs: SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} run: | sentry-cli debug-files upload -o ${{ env.SENTRY_ORG }} -p ${{ env.SENTRY_PROJECT }} target/${{ matrix.settings.target }}/release/cap_desktop.pdb - - done: - needs: [draft, build] - runs-on: ubuntu-latest - permissions: - contents: write - id-token: write - steps: - - name: Send Discord notification - if: ${{ inputs.interactionId != '' }} - uses: actions/github-script@v7 - with: - script: | - async function main() { - const token = await core.getIDToken("cap-discord-bot"); - const cnReleaseId = JSON.parse(`${{ needs.draft.outputs.cn_release_stdout }}`).id; - - const resp = await fetch("https://cap-discord-bot.brendonovich.workers.dev/github-workflow", { - method: "POST", - body: JSON.stringify({ - type: "release-done", - interactionId: "${{ inputs.interactionId }}", - version: "${{ needs.draft.outputs.version }}", - releaseUrl: "${{ needs.draft.outputs.gh_release_url }}", - cnReleaseId - }), - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${token}`, - } - }); - - if(resp.status !== 200) throw new Error(await resp.text()); - } - - main(); diff --git a/FLYOVER_FEATURE_PLAN.md b/FLYOVER_FEATURE_PLAN.md new file mode 100644 index 0000000000..6c7b472d7a --- /dev/null +++ b/FLYOVER_FEATURE_PLAN.md @@ -0,0 +1,1808 @@ +# Flyover Camera Feature - Architecture Analysis & Implementation Plan + +## Table of Contents +1. [Project Overview](#project-overview) +2. [Architecture Analysis](#architecture-analysis) +3. [Current System Deep Dive](#current-system-deep-dive) +4. [Feature Requirements](#feature-requirements) +5. [Implementation Plan](#implementation-plan) +6. [Technical Decisions](#technical-decisions) +7. [Risks & Mitigations](#risks--mitigations) + +--- + +## Project Overview + +### Feature Goal +Add a "flyover" camera mode to Cap's instant recordings where the user's camera video dynamically follows cursor movements during both recording preview and playback. Users should be able to toggle between flyover mode (camera follows cursor) and fixed position mode (camera in bottom-right corner). + +### Key Requirements +- **Mode**: Instant mode (not studio mode) +- **Real-time Preview**: Camera follows cursor during recording in desktop app +- **Playback Format**: Separate video tracks (display.mp4, camera.mp4) + cursor.json uploaded to web +- **Editable**: Users can adjust flyover settings after recording +- **Toggle**: Switch between flyover and fixed position modes + +--- + +## Architecture Analysis + +### Current Recording Modes + +#### Instant Mode (`crates/recording/src/instant_recording.rs`) +**Current Behavior**: +- Single-file output: `content/output.mp4` +- Real-time compositing during recording +- Screen + system audio + microphone → single MP4 +- **Camera is NOT recorded** (captured but not encoded) +- Designed for fast sharing with immediate upload + +**Key Code References**: +```rust +// Line 302: Camera explicitly disabled +camera_feed: None + +// Lines 220-229: Single pipeline creation +let output = ScreenCaptureMethod::make_instant_mode_pipeline( + screen_capture, + system_audio, + mic_feed, + output_path.clone(), + output_resolution, + encoder_preferences, +).await? +``` + +**Output Structure**: +``` +project/ + content/ + output.mp4 ← Single composited video + recording-meta.json +``` + +#### Studio Mode (`crates/recording/src/studio_recording.rs`) +**Current Behavior**: +- **Separate streams** for each source +- Screen, camera, microphone, system audio recorded independently +- Multiple segments support (pause/resume) +- **Comprehensive cursor tracking** (position + clicks) +- Post-processing rendering pipeline for final output + +**Key Code References**: +```rust +// Lines 816-892: Separate pipelines for each source +Pipeline { + screen: OutputPipeline, // display.mp4 + camera: Option, // camera.mp4 (separate!) + microphone: Option, + system_audio: Option, + cursor: Option, +} +``` + +**Output Structure**: +``` +project/ + content/ + segments/ + segment-0/ + display.mp4 ← Screen recording + camera.mp4 ← Camera feed (SEPARATE!) + audio-input.ogg ← Microphone + system_audio.ogg ← System audio + cursor.json ← Cursor events + cursors/ + cursor_0.png ← Cursor image assets + cursor_1.png + recording-meta.json + cap-project.json +``` + +### Key Finding: Streams Are Already Separate in Studio Mode ✅ + +Studio mode demonstrates the pattern we need: +1. Screen and camera recorded to separate MP4 files +2. Cursor positions tracked at 100Hz (10ms intervals) +3. Post-processing compositor positions camera over screen +4. All data available for flexible editing + +--- + +## Current System Deep Dive + +### Cursor Tracking System (`crates/recording/src/cursor.rs`) + +**Already Implemented in Studio Mode** ✅ + +**Capture Details**: +- Polling frequency: 10ms (100 Hz) - Line 80 +- Position normalization: 0.0-1.0 coordinates (relative to screen) +- Click tracking: Down/Up events with timestamps +- Cursor image capture: PNG files for different cursor shapes +- Modifier keys: Ctrl, Shift, Alt, etc. + +**Data Structures**: +```rust +pub struct CursorMoveEvent { + cursor_id: String, + time_ms: f64, // Milliseconds since recording start + x: f64, // Normalized 0.0-1.0 + y: f64, // Normalized 0.0-1.0 + active_modifiers: Vec, +} + +pub struct CursorClickEvent { + down: bool, + cursor_num: u8, + cursor_id: String, + time_ms: f64, +} + +pub struct Cursor { + pub file_name: String, // cursor_N.png + pub id: u32, + pub hotspot: XY, // Click point offset + pub shape: Option, +} +``` + +**Output Format** (`cursor.json`): +```json +{ + "moves": [ + { + "active_modifiers": [], + "cursor_id": "0", + "time_ms": 123.45, + "x": 0.5234, + "y": 0.6789 + } + ], + "clicks": [ + { + "down": true, + "cursor_num": 0, + "cursor_id": "0", + "time_ms": 456.78 + } + ] +} +``` + +**Reusability**: Can be directly reused in instant mode with minimal changes. + +### Camera Recording System (`crates/recording/src/feeds/camera.rs`) + +**Actor-Based Architecture**: +```rust +pub struct CameraFeed { + state: State, // Open or Locked + senders: Vec>, + on_ready: Vec>, +} +``` + +**Capture Flow**: +1. Platform-specific camera APIs: + - macOS: AVFoundation (`camera-avfoundation`) + - Windows: Media Foundation (`camera-mediafoundation`) + - Fallback: FFmpeg (`camera-ffmpeg`) +2. Format selection prioritizes: + - Frame rate ≥ 30 FPS + - Resolution < 2000x2000 + - 16:9 aspect ratio +3. Native capture → FFmpeg frames → H.264 encoding +4. Output to separate `camera.mp4` file + +**Studio Mode Integration**: +```rust +// Lines 827-836 in studio_recording.rs +let camera = OptionFuture::from(base_inputs.camera_feed.map(|camera_feed| { + OutputPipeline::builder(dir.join("camera.mp4")) + .with_video::(camera_feed) + .with_timestamps(start_time) + .build::(()) +})) +``` + +### Rendering Pipeline (`crates/rendering/`) + +**GPU-Accelerated Compositor** built on WGPU. + +**Layer Architecture**: +1. **Background Layer** (`background.rs`) - Solid color or gradient +2. **Display Layer** (`display.rs`) - Screen recording video +3. **Camera Layer** (`camera.rs`) - Separate video texture with position/size/opacity control +4. **Cursor Layer** (`cursor.rs`) - Cursor images with interpolated motion +5. **Captions Layer** (`captions.rs`) - AI-generated subtitles + +**Camera Layer Key Features**: +```rust +pub struct CameraLayer { + frame_texture: wgpu::Texture, + uniforms_buffer: wgpu::Buffer, + hidden: bool, +} + +pub fn prepare( + &mut self, + data: Option<(CompositeVideoFrameUniforms, XY, &DecodedFrame)> +) { + // CompositeVideoFrameUniforms includes: + // - position (x, y in screen space) + // - size (width, height) + // - opacity + // - corner_radius +} +``` + +**Cursor Rendering** (`cursor.rs`, `cursor_interpolation.rs`): +- Spring-mass-damper physics for smooth motion +- Motion blur based on velocity +- Click animations (shrink effect) +- Three spring profiles: Default, Snappy (near clicks), Drag (button held) + +**Coordinate Systems** (`coord.rs`): +```rust +pub struct RawDisplayUVSpace; // Normalized 0.0-1.0 +pub struct FrameSpace; // Output video pixels +pub struct ZoomedFrameSpace; // After zoom transform +``` + +### Preview System (`apps/desktop/src/routes/editor/Player.tsx`) + +**Current Implementation**: +- 2D Canvas element (line 426-437) +- Rust GPU renderer generates frames +- Frames sent via IPC as `ImageData` +- Canvas displays with `ctx.putImageData(frame.data, 0, 0)` (line 373) +- `renderFrameEvent` triggers frame generation +- 30-60 FPS rendering + +**Frame Pipeline**: +``` +SolidJS Component → renderFrameEvent (frame number) + ↓ +Rust editor_instance.rs → Handle event + ↓ +cap_rendering crate → GPU composition + ↓ +Return RenderedFrame with ImageData + ↓ +Canvas display +``` + +--- + +## Feature Requirements + +Based on user answers to clarifying questions: + +1. **Recording Mode**: Instant mode (current default) +2. **Live Preview**: Yes - camera follows cursor in real-time during recording +3. **Playback Format**: Separate tracks (display.mp4, camera.mp4, cursor.json) +4. **Post-Recording Editing**: Yes - users can adjust flyover settings after recording + +### Functional Requirements + +**Desktop App (Recording)**: +- Record screen, camera, and cursor position separately +- Real-time preview shows camera following cursor during recording +- Toggle between flyover mode and fixed position mode +- Configurable offset (camera position relative to cursor) +- Smooth motion with spring physics +- Output multiple files: display.mp4, camera.mp4, cursor.json + +**Web App (Playback)**: +- Upload multiple video tracks + cursor data +- Custom player with synchronized video elements +- Camera position calculated from cursor.json in real-time +- Smooth interpolation between cursor events +- Editable flyover settings (offset, smoothing, enable/disable) +- Preview changes without re-rendering video + +**Web App (Editor)**: +- Toggle flyover on/off +- Adjust camera offset from cursor +- Adjust smoothing strength +- Real-time preview of changes +- Save settings to database +- Optional: Re-render video with baked camera positions + +--- + +## Implementation Plan + +### Phase 1: Desktop Recording Infrastructure (Rust) + +#### 1.1 Add Camera Recording to Instant Mode + +**Files to Modify**: +- `crates/recording/src/instant_recording.rs` +- `crates/recording/src/capture_pipeline.rs` + +**Changes**: +1. Update `ActorBuilder` to accept camera feed +```rust +pub fn with_camera_feed(mut self, camera_feed: Arc) -> Self { + self.camera_feed = Some(camera_feed); + self +} +``` + +2. Modify line 302 to use camera feed instead of `None`: +```rust +RecordingBaseInputs { + capture_target: self.capture_target, + capture_system_audio: self.system_audio, + mic_feed: self.mic_feed, + camera_feed: self.camera_feed, // Change from None + ... +} +``` + +3. Update `create_pipeline` to create separate camera output: +```rust +let camera = if let Some(camera_feed) = base_inputs.camera_feed { + Some(OutputPipeline::builder(content_dir.join("camera.mp4")) + .with_video::(camera_feed) + .with_timestamps(start_time) + .build::(()) + .await?) +} else { + None +} +``` + +4. Update `make_instant_mode_pipeline` trait to accept optional camera parameter + +**Output**: Instant recordings produce `display.mp4` and `camera.mp4` separately + +#### 1.2 Add Cursor Tracking to Instant Mode + +**Files to Modify**: +- `crates/recording/src/instant_recording.rs` +- Reuse: `crates/recording/src/cursor.rs` (no changes needed) + +**Changes**: +1. Add cursor recorder to pipeline creation: +```rust +let cursor = if enable_cursor { + let cursor_crop_bounds = target.cursor_crop() + .ok_or_else(|| anyhow!("No cursor bounds"))?; + + let cursor = spawn_cursor_recorder( + cursor_crop_bounds, + display, + content_dir.join("cursors"), + HashMap::new(), // prev_cursors + 0, // next_cursor_id + start_time, + ); + + Some(CursorPipeline { + output_path: content_dir.join("cursor.json"), + actor: cursor, + }) +} else { + None +} +``` + +2. Add cursor directory creation +3. Update Pipeline struct to include cursor + +**Output**: `cursor.json` with 100Hz position data, cursor images in `cursors/` directory + +#### 1.3 Update Metadata Format + +**Files to Modify**: +- `crates/project/src/meta.rs` + +**Changes**: +1. Add new variant to `InstantRecordingMeta`: +```rust +pub enum InstantRecordingMeta { + InProgress { recording: bool }, + Failed { error: String }, + Complete { + fps: u32, + sample_rate: Option + }, + // NEW: Multi-track format + MultiTrack { + display: VideoTrackMeta, + camera: Option, + mic: Option, + system_audio: Option, + cursor: Option, + fps: u32, + } +} + +pub struct VideoTrackMeta { + pub path: RelativePathBuf, + pub fps: u32, + pub resolution: (u32, u32), + pub start_time: f64, +} +``` + +2. Update serialization/deserialization +3. Maintain backward compatibility with legacy `output.mp4` format + +**Output Structure**: +``` +project/ + content/ + display.mp4 ← Screen only + camera.mp4 ← Camera only + audio-input.ogg ← Mic audio + system_audio.ogg ← System audio (optional) + cursor.json ← Cursor data + cursors/ ← Cursor images + cursor_0.png + recording-meta.json ← Updated format +``` + +--- + +### Phase 2: Real-time Preview Compositor (Rust + SolidJS) + +#### 2.1 Camera Follow Position Calculator + +**New File**: `crates/rendering/src/camera_follow.rs` + +**Implementation**: +```rust +use crate::coord::{Coord, RawDisplayUVSpace}; +use crate::spring_mass_damper::{SpringMassDamperSimulationConfig, SpringMassDamper}; + +pub struct CameraFollowConfig { + pub enabled: bool, + pub offset: XY, // Offset from cursor (e.g., 150px right, 150px down) + pub camera_size: XY, // Camera dimensions + pub smoothing: SpringMassDamperSimulationConfig, + pub boundary_padding: f64, // Padding from screen edges +} + +pub struct CameraFollowState { + position: SpringMassDamper>, + config: CameraFollowConfig, +} + +impl CameraFollowState { + pub fn update( + &mut self, + cursor_pos: Coord, + delta_time: f64, + ) -> Coord { + let target_pos = self.calculate_target_position(cursor_pos); + self.position.update(target_pos, delta_time); + + // Apply boundary constraints + self.constrain_to_bounds(self.position.position()) + } + + fn calculate_target_position(&self, cursor_pos: Coord) -> XY { + // Camera position = cursor position + offset + let target = XY { + x: cursor_pos.x() + self.config.offset.x, + y: cursor_pos.y() + self.config.offset.y, + }; + target + } + + fn constrain_to_bounds(&self, pos: XY) -> Coord { + // Ensure camera stays within screen bounds (0.0 - 1.0) + // with padding for camera size + let constrained = XY { + x: pos.x.clamp( + self.config.boundary_padding, + 1.0 - self.config.camera_size.x - self.config.boundary_padding + ), + y: pos.y.clamp( + self.config.boundary_padding, + 1.0 - self.config.camera_size.y - self.config.boundary_padding + ), + }; + Coord::from_xy(constrained) + } +} +``` + +**Features**: +- Spring-mass-damper smoothing (reuse existing implementation) +- Configurable offset from cursor +- Boundary checking to keep camera on-screen +- Smooth transitions + +#### 2.2 Extend GPU Renderer for Live Camera Overlay + +**Files to Modify**: +- `crates/rendering/src/lib.rs` +- `crates/rendering/src/layers/camera.rs` + +**Changes to `lib.rs`**: +1. Add preview mode flag to renderer +2. Accept live camera feed and cursor position +3. Calculate camera position using `CameraFollowState` +4. Update `ProjectUniforms` to include camera follow config + +**Changes to `camera.rs`**: +1. Support real-time camera feed (not just pre-recorded video) +2. Accept dynamic position parameter +3. Update uniforms buffer with new position each frame + +**Implementation**: +```rust +// In rendering loop +let cursor_position = get_current_cursor_position(); +let camera_position = camera_follow_state.update(cursor_position, delta_time); + +// Prepare camera layer with dynamic position +camera_layer.prepare(Some(( + CompositeVideoFrameUniforms { + position: camera_position, + size: camera_config.size, + opacity: 1.0, + corner_radius: camera_config.corner_radius, + }, + camera_frame.size, + &camera_frame, +))); +``` + +**Output**: Composited frames with camera overlay at cursor-following position + +#### 2.3 Update Preview Frame Generation + +**Files to Modify**: +- `apps/desktop/src-tauri/src/editor_instance.rs` +- `apps/desktop/src-tauri/src/recording.rs` + +**Changes**: +1. Pass camera feed to renderer during recording +2. Pass current cursor position to renderer +3. Enable camera overlay for instant mode preview +4. Stream composited frames to frontend at 30-60 FPS + +**IPC Event**: +```rust +#[derive(Serialize, Type, tauri_specta::Event, Debug, Clone)] +pub struct PreviewFrameReady { + frame: ImageData, + timestamp: f64, +} +``` + +#### 2.4 Desktop UI Controls + +**Files to Modify**: +- `apps/desktop/src/routes/editor/Player.tsx` +- New: `apps/desktop/src/components/CameraFollowControls.tsx` + +**UI Components**: +1. Toggle switch: "Flyover Mode" vs "Fixed Position" +2. Offset controls: + - X offset slider (-500 to 500 pixels) + - Y offset slider (-500 to 500 pixels) +3. Smoothing strength slider (0.1 to 1.0) +4. Camera size dropdown (Small, Medium, Large) +5. Preview indicator showing camera will follow cursor + +**State Management**: +```typescript +const [cameraMode, setCameraMode] = createSignal<'flyover' | 'fixed'>('fixed'); +const [offset, setOffset] = createSignal({ x: 150, y: 150 }); +const [smoothing, setSmoothing] = createSignal(0.5); + +// Send config to Rust via IPC +createEffect(() => { + commands.updateCameraFollowConfig({ + enabled: cameraMode() === 'flyover', + offset: offset(), + smoothing: smoothing(), + }); +}); +``` + +**Real-time Preview**: +- Canvas receives composited frames from Rust +- Settings updates trigger immediate re-render +- Visual feedback of camera position + +--- + +### Phase 3: Web Upload & Storage + +#### 3.1 Multi-File Upload Pipeline + +**Files to Modify**: +- `apps/web/actions/video/upload.ts` (or create new Server Action) +- `apps/web/lib/s3.ts` + +**Changes**: +1. Create upload function for multiple files: +```typescript +"use server"; + +export async function uploadMultiTrackVideo(data: { + videoId: string; + displayFile: File; + cameraFile: File; + cursorData: string; // JSON string + metadata: VideoMetadata; +}) { + const s3 = getS3Client(); + + // Upload files concurrently + const [displayUrl, cameraUrl, cursorUrl] = await Promise.all([ + s3.upload({ + key: `videos/${data.videoId}/display.mp4`, + file: data.displayFile, + }), + s3.upload({ + key: `videos/${data.videoId}/camera.mp4`, + file: data.cameraFile, + }), + s3.upload({ + key: `videos/${data.videoId}/cursor.json`, + file: new Blob([data.cursorData], { type: 'application/json' }), + }), + ]); + + // Update database with all URLs + await db.update(videos).set({ + videoPath: displayUrl, + cameraVideoPath: cameraUrl, + cursorDataPath: cursorUrl, + status: 'ready', + }).where(eq(videos.id, data.videoId)); + + return { success: true, videoId: data.videoId }; +} +``` + +2. Progress tracking for all uploads +3. Atomic commit (only mark ready when all files uploaded) +4. Error handling and retry logic + +#### 3.2 Database Schema Updates + +**Files to Modify**: +- `packages/database/schema.ts` + +**Schema Changes**: +```typescript +export const videos = mysqlTable("videos", { + id: varchar("id", { length: 26 }).primaryKey(), + + // Existing fields + videoPath: text("videoPath").notNull(), + + // NEW: Multi-track support + cameraVideoPath: text("cameraVideoPath"), // camera.mp4 URL + cursorDataPath: text("cursorDataPath"), // cursor.json URL + recordingMode: varchar("recordingMode", { + length: 20, + enum: ["instant", "studio"] + }).default("instant"), + + // Camera follow settings + cameraFollowEnabled: boolean("cameraFollowEnabled").default(false), + cameraFollowOffset: json("cameraFollowOffset").$type<{ x: number; y: number }>(), + cameraFollowSmoothing: float("cameraFollowSmoothing").default(0.5), + + // ... other existing fields +}); +``` + +**Migration Script**: +```sql +ALTER TABLE videos + ADD COLUMN cameraVideoPath TEXT, + ADD COLUMN cursorDataPath TEXT, + ADD COLUMN recordingMode VARCHAR(20) DEFAULT 'instant', + ADD COLUMN cameraFollowEnabled BOOLEAN DEFAULT FALSE, + ADD COLUMN cameraFollowOffset JSON, + ADD COLUMN cameraFollowSmoothing FLOAT DEFAULT 0.5; +``` + +**Run Migration**: +```bash +pnpm db:generate +pnpm db:push +``` + +#### 3.3 S3 Storage Structure + +**New Structure**: +``` +s3://cap-bucket/ + videos/ + {videoId}/ + display.mp4 ← Screen recording + camera.mp4 ← Camera recording + cursor.json ← Cursor position data + thumbnail.jpg ← Thumbnail (existing) +``` + +**Backward Compatibility**: +- Old recordings: `videos/{videoId}.mp4` (single file) +- New recordings: `videos/{videoId}/display.mp4` (multi-track) +- Check for `cameraVideoPath` to determine format + +--- + +### Phase 4: Web Video Player + +#### 4.1 Multi-Track Player Component + +**New File**: `apps/web/components/VideoPlayer/MultiTrackPlayer.tsx` + +**Implementation**: +```typescript +"use client"; + +import { useEffect, useRef, useState } from "react"; +import { useCursorPositioning } from "./useCursorPositioning"; + +interface MultiTrackPlayerProps { + displayVideoUrl: string; + cameraVideoUrl: string; + cursorDataUrl: string; + cameraFollowConfig: { + enabled: boolean; + offset: { x: number; y: number }; + smoothing: number; + }; +} + +export function MultiTrackPlayer({ + displayVideoUrl, + cameraVideoUrl, + cursorDataUrl, + cameraFollowConfig, +}: MultiTrackPlayerProps) { + const displayRef = useRef(null); + const cameraRef = useRef(null); + const containerRef = useRef(null); + + const [currentTime, setCurrentTime] = useState(0); + const [isPlaying, setIsPlaying] = useState(false); + + // Load and process cursor data + const { getCameraPosition } = useCursorPositioning( + cursorDataUrl, + cameraFollowConfig + ); + + // Sync both videos + useEffect(() => { + const display = displayRef.current; + const camera = cameraRef.current; + if (!display || !camera) return; + + const syncVideos = () => { + const timeDiff = Math.abs(display.currentTime - camera.currentTime); + + // If drift > 50ms, correct it + if (timeDiff > 0.05) { + camera.currentTime = display.currentTime; + } + + setCurrentTime(display.currentTime); + }; + + display.addEventListener('timeupdate', syncVideos); + display.addEventListener('play', () => { + camera.play(); + setIsPlaying(true); + }); + display.addEventListener('pause', () => { + camera.pause(); + setIsPlaying(false); + }); + display.addEventListener('seeked', () => { + camera.currentTime = display.currentTime; + }); + + return () => { + display.removeEventListener('timeupdate', syncVideos); + }; + }, []); + + // Update camera position based on cursor data + useEffect(() => { + if (!cameraRef.current || !containerRef.current) return; + + let animationFrame: number; + + const updatePosition = () => { + const position = getCameraPosition(currentTime); + + if (position && cameraRef.current) { + const container = containerRef.current!; + const containerRect = container.getBoundingClientRect(); + + // Convert normalized position (0-1) to pixels + const x = position.x * containerRect.width; + const y = position.y * containerRect.height; + + cameraRef.current.style.transform = `translate(${x}px, ${y}px)`; + } + + animationFrame = requestAnimationFrame(updatePosition); + }; + + if (isPlaying) { + updatePosition(); + } + + return () => cancelAnimationFrame(animationFrame); + }, [currentTime, isPlaying, getCameraPosition]); + + return ( +
+ {/* Screen video (base layer) */} +
+ ); +} +``` + +**Key Features**: +- Two synchronized `