diff --git a/.claude/settings.local.json b/.claude/settings.local.json index 0f864d7abe..f0e207bf26 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -50,7 +50,11 @@ "Bash(cargo build:*)", "Bash(footprint:*)", "Bash(RUST_LOG=info,cap_recording=debug ./target/release/examples/memory-leak-detector:*)", - "Bash(git rm:*)" + "Bash(git rm:*)", + "Bash(./target/release/examples/decode-benchmark:*)", + "Bash(RUST_LOG=warn ./target/release/examples/decode-benchmark:*)", + "Bash(git mv:*)", + "Bash(xargs cat:*)" ], "deny": [], "ask": [] diff --git a/Cargo.lock b/Cargo.lock index 4aa78c2b5f..90a9a5dd1c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1546,6 +1546,8 @@ dependencies = [ "indexmap 2.11.4", "inquire", "kameo", + "libc", + "libproc", "objc", "objc2-app-kit", "relative-path", @@ -4870,6 +4872,17 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" +[[package]] +name = "libproc" +version = "0.14.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e78a09b56be5adbcad5aa1197371688dc6bb249a26da3bca2011ee2fb987ebfb" +dependencies = [ + "bindgen 0.70.1", + "errno", + "libc", +] + [[package]] name = "libredox" version = "0.1.10" diff --git a/apps/desktop/src-tauri/src/export.rs b/apps/desktop/src-tauri/src/export.rs index b9226028c2..e1c315f592 100644 --- a/apps/desktop/src-tauri/src/export.rs +++ b/apps/desktop/src-tauri/src/export.rs @@ -1,9 +1,15 @@ +use crate::editor_window::WindowEditorInstance; use crate::{FramesRendered, get_video_metadata}; use cap_export::ExporterBase; -use cap_project::RecordingMeta; -use serde::Deserialize; +use cap_project::{RecordingMeta, XY}; +use cap_rendering::{ + FrameRenderer, ProjectRecordingsMeta, ProjectUniforms, RenderSegment, RenderVideoConstants, + RendererLayers, +}; +use image::codecs::jpeg::JpegEncoder; +use serde::{Deserialize, Serialize}; use specta::Type; -use std::path::PathBuf; +use std::{path::PathBuf, sync::Arc}; use tracing::{info, instrument}; #[derive(Deserialize, Clone, Copy, Debug, Type)] @@ -164,3 +170,281 @@ pub async fn get_export_estimates( estimated_size_mb, }) } + +#[derive(Debug, Deserialize, Type)] +pub struct ExportPreviewSettings { + pub fps: u32, + pub resolution_base: XY, + pub compression_bpp: f32, +} + +#[derive(Debug, Serialize, Type)] +pub struct ExportPreviewResult { + pub jpeg_base64: String, + pub estimated_size_mb: f64, + pub actual_width: u32, + pub actual_height: u32, + pub frame_render_time_ms: f64, + pub total_frames: u32, +} + +fn bpp_to_jpeg_quality(bpp: f32) -> u8 { + ((bpp - 0.04) / (0.3 - 0.04) * (95.0 - 40.0) + 40.0).clamp(40.0, 95.0) as u8 +} + +#[tauri::command] +#[specta::specta] +#[instrument(skip_all)] +pub async fn generate_export_preview( + project_path: PathBuf, + frame_time: f64, + settings: ExportPreviewSettings, +) -> Result { + use base64::{Engine, engine::general_purpose::STANDARD}; + use cap_editor::create_segments; + use std::time::Instant; + + let recording_meta = RecordingMeta::load_for_project(&project_path) + .map_err(|e| format!("Failed to load recording meta: {e}"))?; + + let cap_project::RecordingMetaInner::Studio(studio_meta) = &recording_meta.inner else { + return Err("Cannot preview non-studio recordings".to_string()); + }; + + let project_config = recording_meta.project_config(); + + let recordings = Arc::new( + ProjectRecordingsMeta::new(&recording_meta.project_path, studio_meta) + .map_err(|e| format!("Failed to load recordings: {e}"))?, + ); + + let render_constants = Arc::new( + RenderVideoConstants::new( + &recordings.segments, + recording_meta.clone(), + studio_meta.clone(), + ) + .await + .map_err(|e| format!("Failed to create render constants: {e}"))?, + ); + + let segments = create_segments(&recording_meta, studio_meta) + .await + .map_err(|e| format!("Failed to create segments: {e}"))?; + + let render_segments: Vec = segments + .iter() + .map(|s| RenderSegment { + cursor: s.cursor.clone(), + decoders: s.decoders.clone(), + }) + .collect(); + + let Some((segment_time, segment)) = project_config.get_segment_time(frame_time) else { + return Err("Frame time is outside video duration".to_string()); + }; + + let render_segment = &render_segments[segment.recording_clip as usize]; + let clip_config = project_config + .clips + .iter() + .find(|v| v.index == segment.recording_clip); + + let render_start = Instant::now(); + + let segment_frames = render_segment + .decoders + .get_frames( + segment_time as f32, + !project_config.camera.hide, + clip_config.map(|v| v.offsets).unwrap_or_default(), + ) + .await + .ok_or_else(|| "Failed to decode frame".to_string())?; + + let frame_number = (frame_time * settings.fps as f64).floor() as u32; + + let uniforms = ProjectUniforms::new( + &render_constants, + &project_config, + frame_number, + settings.fps, + settings.resolution_base, + &render_segment.cursor, + &segment_frames, + ); + + let mut frame_renderer = FrameRenderer::new(&render_constants); + let mut layers = RendererLayers::new_with_options( + &render_constants.device, + &render_constants.queue, + render_constants.is_software_adapter, + ); + + let frame = frame_renderer + .render( + segment_frames, + uniforms, + &render_segment.cursor, + &mut layers, + ) + .await + .map_err(|e| format!("Failed to render frame: {e}"))?; + + let frame_render_time_ms = render_start.elapsed().as_secs_f64() * 1000.0; + + let width = frame.width; + let height = frame.height; + + let rgb_data: Vec = frame + .data + .chunks(frame.padded_bytes_per_row as usize) + .flat_map(|row| { + row[0..(frame.width * 4) as usize] + .chunks(4) + .flat_map(|chunk| [chunk[0], chunk[1], chunk[2]]) + }) + .collect(); + + let jpeg_quality = bpp_to_jpeg_quality(settings.compression_bpp); + let mut jpeg_buffer = Vec::new(); + { + let mut encoder = JpegEncoder::new_with_quality(&mut jpeg_buffer, jpeg_quality); + encoder + .encode(&rgb_data, width, height, image::ExtendedColorType::Rgb8) + .map_err(|e| format!("Failed to encode JPEG: {e}"))?; + } + + let jpeg_base64 = STANDARD.encode(&jpeg_buffer); + + let total_pixels = (settings.resolution_base.x * settings.resolution_base.y) as f64; + let fps_f64 = settings.fps as f64; + + let metadata = get_video_metadata(project_path.clone()).await?; + let duration_seconds = if let Some(timeline) = &project_config.timeline { + timeline.segments.iter().map(|s| s.duration()).sum() + } else { + metadata.duration + }; + let total_frames = (duration_seconds * fps_f64).ceil() as u32; + + let video_bitrate = total_pixels * settings.compression_bpp as f64 * fps_f64; + let audio_bitrate = 192_000.0; + let total_bitrate = video_bitrate + audio_bitrate; + let estimated_size_mb = (total_bitrate * duration_seconds) / (8.0 * 1024.0 * 1024.0); + + Ok(ExportPreviewResult { + jpeg_base64, + estimated_size_mb, + actual_width: width, + actual_height: height, + frame_render_time_ms, + total_frames, + }) +} + +#[tauri::command] +#[specta::specta] +#[instrument(skip_all)] +pub async fn generate_export_preview_fast( + editor: WindowEditorInstance, + frame_time: f64, + settings: ExportPreviewSettings, +) -> Result { + use base64::{Engine, engine::general_purpose::STANDARD}; + use std::time::Instant; + + let project_config = editor.project_config.1.borrow().clone(); + + let Some((segment_time, segment)) = project_config.get_segment_time(frame_time) else { + return Err("Frame time is outside video duration".to_string()); + }; + + let segment_media = &editor.segment_medias[segment.recording_clip as usize]; + let clip_config = project_config + .clips + .iter() + .find(|v| v.index == segment.recording_clip); + + let render_start = Instant::now(); + + let segment_frames = segment_media + .decoders + .get_frames( + segment_time as f32, + !project_config.camera.hide, + clip_config.map(|v| v.offsets).unwrap_or_default(), + ) + .await + .ok_or_else(|| "Failed to decode frame".to_string())?; + + let frame_number = (frame_time * settings.fps as f64).floor() as u32; + + let uniforms = ProjectUniforms::new( + &editor.render_constants, + &project_config, + frame_number, + settings.fps, + settings.resolution_base, + &segment_media.cursor, + &segment_frames, + ); + + let mut frame_renderer = FrameRenderer::new(&editor.render_constants); + let mut layers = RendererLayers::new_with_options( + &editor.render_constants.device, + &editor.render_constants.queue, + editor.render_constants.is_software_adapter, + ); + + let frame = frame_renderer + .render(segment_frames, uniforms, &segment_media.cursor, &mut layers) + .await + .map_err(|e| format!("Failed to render frame: {e}"))?; + + let frame_render_time_ms = render_start.elapsed().as_secs_f64() * 1000.0; + + let width = frame.width; + let height = frame.height; + + let rgb_data: Vec = frame + .data + .chunks(frame.padded_bytes_per_row as usize) + .flat_map(|row| { + row[0..(frame.width * 4) as usize] + .chunks(4) + .flat_map(|chunk| [chunk[0], chunk[1], chunk[2]]) + }) + .collect(); + + let jpeg_quality = bpp_to_jpeg_quality(settings.compression_bpp); + let mut jpeg_buffer = Vec::new(); + { + let mut encoder = JpegEncoder::new_with_quality(&mut jpeg_buffer, jpeg_quality); + encoder + .encode(&rgb_data, width, height, image::ExtendedColorType::Rgb8) + .map_err(|e| format!("Failed to encode JPEG: {e}"))?; + } + + let jpeg_base64 = STANDARD.encode(&jpeg_buffer); + + let total_pixels = (settings.resolution_base.x * settings.resolution_base.y) as f64; + let fps_f64 = settings.fps as f64; + + let duration_seconds = editor.recordings.duration(); + let total_frames = (duration_seconds * fps_f64).ceil() as u32; + + let video_bitrate = total_pixels * settings.compression_bpp as f64 * fps_f64; + let audio_bitrate = 192_000.0; + let total_bitrate = video_bitrate + audio_bitrate; + let estimated_size_mb = (total_bitrate * duration_seconds) / (8.0 * 1024.0 * 1024.0); + + Ok(ExportPreviewResult { + jpeg_base64, + estimated_size_mb, + actual_width: width, + actual_height: height, + frame_render_time_ms, + total_frames, + }) +} diff --git a/apps/desktop/src-tauri/src/lib.rs b/apps/desktop/src-tauri/src/lib.rs index e914955174..2863cf9998 100644 --- a/apps/desktop/src-tauri/src/lib.rs +++ b/apps/desktop/src-tauri/src/lib.rs @@ -90,7 +90,7 @@ use tauri_plugin_notification::{NotificationExt, PermissionState}; use tauri_plugin_opener::OpenerExt; use tauri_plugin_shell::ShellExt; use tauri_specta::Event; -use tokio::sync::{RwLock, oneshot}; +use tokio::sync::{RwLock, oneshot, watch}; use tracing::*; use upload::{create_or_get_video, upload_image, upload_video}; use web_api::AuthedApiError; @@ -106,6 +106,43 @@ use crate::{ }; use crate::{recording::start_recording, upload::build_video_meta}; +type FinalizingRecordingsMap = + std::collections::HashMap, watch::Receiver)>; + +#[derive(Default)] +pub struct FinalizingRecordings { + recordings: std::sync::Mutex, +} + +impl FinalizingRecordings { + pub fn start_finalizing(&self, path: PathBuf) -> watch::Receiver { + let mut recordings = self + .recordings + .lock() + .expect("FinalizingRecordings mutex poisoned"); + let (tx, rx) = watch::channel(false); + recordings.insert(path, (tx, rx.clone())); + rx + } + + pub fn finish_finalizing(&self, path: &Path) { + let mut recordings = self + .recordings + .lock() + .expect("FinalizingRecordings mutex poisoned"); + if let Some((tx, _)) = recordings.remove(path) + && tx.send(true).is_err() + { + debug!("Finalizing receiver dropped for path: {:?}", path); + } + } + + pub fn is_finalizing(&self, path: &Path) -> Option> { + let recordings = self.recordings.lock().unwrap(); + recordings.get(path).map(|(_, rx)| rx.clone()) + } +} + #[allow(clippy::large_enum_variant)] pub enum RecordingState { None, @@ -2366,6 +2403,8 @@ pub async fn run(recording_logging_handle: LoggingHandle, logs_dir: PathBuf) { get_current_recording, export::export_video, export::get_export_estimates, + export::generate_export_preview, + export::generate_export_preview_fast, copy_file_to_path, copy_video_to_clipboard, copy_screenshot_to_clipboard, @@ -2597,6 +2636,7 @@ pub async fn run(recording_logging_handle: LoggingHandle, logs_dir: PathBuf) { app.manage(http_client::HttpClient::default()); app.manage(http_client::RetryableHttpClient::default()); app.manage(PendingScreenshots::default()); + app.manage(FinalizingRecordings::default()); gpu_context::prewarm_gpu(); @@ -3142,6 +3182,8 @@ async fn create_editor_instance_impl( ) -> Result, String> { let app = app.clone(); + wait_for_recording_ready(&app, &path).await?; + let instance = { let app = app.clone(); EditorInstance::new( @@ -3170,6 +3212,39 @@ async fn create_editor_instance_impl( Ok(instance) } +async fn wait_for_recording_ready(app: &AppHandle, path: &Path) -> Result<(), String> { + let finalizing_state = app.state::(); + + if let Some(mut rx) = finalizing_state.is_finalizing(path) { + info!("Recording is being finalized, waiting for completion..."); + rx.wait_for(|&ready| ready) + .await + .map_err(|_| "Finalization was cancelled".to_string())?; + info!("Recording finalization completed"); + return Ok(()); + } + + let meta = match RecordingMeta::load_for_project(path) { + Ok(meta) => meta, + Err(e) => { + return Err(format!("Failed to load recording meta: {e}")); + } + }; + + if let Some(studio_meta) = meta.studio_meta() + && recording::needs_fragment_remux(path, studio_meta) + { + info!("Recording needs remux (crash recovery), starting remux..."); + let path = path.to_path_buf(); + tokio::task::spawn_blocking(move || recording::remux_fragmented_recording(&path)) + .await + .map_err(|e| format!("Remux task panicked: {e}"))??; + info!("Crash recovery remux completed"); + } + + Ok(()) +} + fn recordings_path(app: &AppHandle) -> PathBuf { let path = app.path().app_data_dir().unwrap().join("recordings"); std::fs::create_dir_all(&path).unwrap_or_default(); diff --git a/apps/desktop/src-tauri/src/recording.rs b/apps/desktop/src-tauri/src/recording.rs index a796749932..4265cbcdf1 100644 --- a/apps/desktop/src-tauri/src/recording.rs +++ b/apps/desktop/src-tauri/src/recording.rs @@ -52,8 +52,8 @@ use crate::camera::{CameraPreviewManager, CameraPreviewShape}; use crate::general_settings; use crate::web_api::AuthedApiError; use crate::{ - App, CurrentRecordingChanged, MutableState, NewStudioRecordingAdded, RecordingState, - RecordingStopped, VideoUploadInfo, + App, CurrentRecordingChanged, FinalizingRecordings, MutableState, NewStudioRecordingAdded, + RecordingState, RecordingStopped, VideoUploadInfo, api::PresignedS3PutRequestMethod, audio::AppSounds, auth::AuthStore, @@ -1379,22 +1379,75 @@ async fn handle_recording_finish( .map_err(|e| format!("Failed to save recording meta: {e}"))?; } - let updated_studio_meta = if needs_fragment_remux(&recording_dir, &recording.meta) { - info!("Recording has fragments that need remuxing"); - if let Err(e) = remux_fragmented_recording(&recording_dir) { - error!("Failed to remux fragmented recording: {e}"); - return Err(format!("Failed to remux fragmented recording: {e}")); + let needs_remux = needs_fragment_remux(&recording_dir, &recording.meta); + + if needs_remux { + info!("Recording has fragments that need remuxing - opening editor immediately"); + + let finalizing_state = app.state::(); + finalizing_state.start_finalizing(recording_dir.clone()); + + let post_behaviour = GeneralSettingsStore::get(app) + .ok() + .flatten() + .map(|v| v.post_studio_recording_behaviour) + .unwrap_or(PostStudioRecordingBehaviour::OpenEditor); + + match post_behaviour { + PostStudioRecordingBehaviour::OpenEditor => { + let _ = ShowCapWindow::Editor { + project_path: recording_dir.clone(), + } + .show(app) + .await; + } + PostStudioRecordingBehaviour::ShowOverlay => { + let _ = ShowCapWindow::RecordingsOverlay.show(app).await; + + let app_clone = AppHandle::clone(app); + let recording_dir_clone = recording_dir.clone(); + tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(1000)).await; + let _ = NewStudioRecordingAdded { + path: recording_dir_clone, + } + .emit(&app_clone); + }); + } } - let updated_meta = RecordingMeta::load_for_project(&recording_dir) - .map_err(|e| format!("Failed to reload recording meta: {e}"))?; - updated_meta - .studio_meta() - .ok_or_else(|| "Expected studio meta after remux".to_string())? - .clone() - } else { - recording.meta.clone() - }; + AppSounds::StopRecording.play(); + + let app = app.clone(); + let recording_dir_for_finalize = recording_dir.clone(); + let screenshots_dir = screenshots_dir.clone(); + let default_preset = PresetsStore::get_default_preset(&app) + .ok() + .flatten() + .map(|p| p.config); + + tokio::spawn(async move { + let result = finalize_studio_recording( + &app, + recording_dir_for_finalize.clone(), + screenshots_dir, + recording, + default_preset, + ) + .await; + + if let Err(e) = result { + error!("Failed to finalize recording: {e}"); + } + + app.state::() + .finish_finalizing(&recording_dir_for_finalize); + }); + + return Ok(()); + } + + let updated_studio_meta = recording.meta.clone(); let display_output_path = match &updated_studio_meta { StudioRecordingMeta::SingleSegment { segment } => { @@ -1588,6 +1641,72 @@ async fn handle_recording_finish( Ok(()) } +async fn finalize_studio_recording( + app: &AppHandle, + recording_dir: PathBuf, + screenshots_dir: PathBuf, + recording: cap_recording::studio_recording::CompletedRecording, + default_preset: Option, +) -> Result<(), String> { + info!("Starting background finalization for recording"); + + let recording_dir_for_remux = recording_dir.clone(); + let remux_result = + tokio::task::spawn_blocking(move || remux_fragmented_recording(&recording_dir_for_remux)) + .await + .map_err(|e| format!("Remux task panicked: {e}"))?; + + if let Err(e) = remux_result { + error!("Failed to remux fragmented recording: {e}"); + return Err(format!("Failed to remux fragmented recording: {e}")); + } + + let updated_meta = RecordingMeta::load_for_project(&recording_dir) + .map_err(|e| format!("Failed to reload recording meta: {e}"))?; + let updated_studio_meta = updated_meta + .studio_meta() + .ok_or_else(|| "Expected studio meta after remux".to_string())? + .clone(); + + let display_output_path = match &updated_studio_meta { + StudioRecordingMeta::SingleSegment { segment } => { + segment.display.path.to_path(&recording_dir) + } + StudioRecordingMeta::MultipleSegments { inner, .. } => { + inner.segments[0].display.path.to_path(&recording_dir) + } + }; + + let display_screenshot = screenshots_dir.join("display.jpg"); + tokio::spawn(create_screenshot( + display_output_path, + display_screenshot, + None, + )); + + let recordings = ProjectRecordingsMeta::new(&recording_dir, &updated_studio_meta) + .map_err(|e| format!("Failed to create project recordings meta: {e}"))?; + + let config = project_config_from_recording( + app, + &cap_recording::studio_recording::CompletedRecording { + project_path: recording.project_path, + meta: updated_studio_meta, + cursor_data: recording.cursor_data, + }, + &recordings, + default_preset, + ); + + config + .write(&recording_dir) + .map_err(|e| format!("Failed to write project config: {e}"))?; + + info!("Background finalization completed for recording"); + + Ok(()) +} + /// Core logic for generating zoom segments based on mouse click events. /// This is an experimental feature that automatically creates zoom effects /// around user interactions to highlight important moments. @@ -1873,7 +1992,7 @@ fn project_config_from_recording( config } -fn needs_fragment_remux(recording_dir: &Path, meta: &StudioRecordingMeta) -> bool { +pub fn needs_fragment_remux(recording_dir: &Path, meta: &StudioRecordingMeta) -> bool { let StudioRecordingMeta::MultipleSegments { inner, .. } = meta else { return false; }; @@ -1888,7 +2007,7 @@ fn needs_fragment_remux(recording_dir: &Path, meta: &StudioRecordingMeta) -> boo false } -fn remux_fragmented_recording(recording_dir: &Path) -> Result<(), String> { +pub fn remux_fragmented_recording(recording_dir: &Path) -> Result<(), String> { let meta = RecordingMeta::load_for_project(recording_dir) .map_err(|e| format!("Failed to load recording meta: {e}"))?; @@ -1924,10 +2043,12 @@ fn analyze_recording_for_remux( for (index, segment) in inner.segments.iter().enumerate() { let display_path = segment.display.path.to_path(project_path); - let display_fragments = if display_path.is_dir() { - find_fragments_in_dir(&display_path) + let (display_fragments, display_init_segment) = if display_path.is_dir() { + let frags = find_fragments_in_dir(&display_path); + let init = display_path.join("init.mp4"); + (frags, if init.exists() { Some(init) } else { None }) } else if display_path.exists() { - vec![display_path] + (vec![display_path], None) } else { continue; }; @@ -1936,17 +2057,27 @@ fn analyze_recording_for_remux( continue; } - let camera_fragments = segment.camera.as_ref().and_then(|cam| { - let cam_path = cam.path.to_path(project_path); - if cam_path.is_dir() { - let frags = find_fragments_in_dir(&cam_path); - if frags.is_empty() { None } else { Some(frags) } - } else if cam_path.exists() { - Some(vec![cam_path]) - } else { - None - } - }); + let (camera_fragments, camera_init_segment) = segment + .camera + .as_ref() + .map(|cam| { + let cam_path = cam.path.to_path(project_path); + if cam_path.is_dir() { + let frags = find_fragments_in_dir(&cam_path); + let init = cam_path.join("init.mp4"); + let init_seg = if init.exists() { Some(init) } else { None }; + if frags.is_empty() { + (None, None) + } else { + (Some(frags), init_seg) + } + } else if cam_path.exists() { + (Some(vec![cam_path]), None) + } else { + (None, None) + } + }) + .unwrap_or((None, None)); let cursor_path = segment .cursor @@ -1981,7 +2112,9 @@ fn analyze_recording_for_remux( recoverable_segments.push(RecoverableSegment { index: index as u32, display_fragments, + display_init_segment, camera_fragments, + camera_init_segment, mic_fragments, system_audio_fragments, cursor_path, diff --git a/apps/desktop/src/routes/editor/Editor.tsx b/apps/desktop/src/routes/editor/Editor.tsx index 90f2b96d74..dd2bba4421 100644 --- a/apps/desktop/src/routes/editor/Editor.tsx +++ b/apps/desktop/src/routes/editor/Editor.tsx @@ -14,6 +14,7 @@ import { createSignal, Match, on, + onCleanup, Show, Switch, } from "solid-js"; @@ -40,19 +41,11 @@ import { useEditorContext, useEditorInstanceContext, } from "./context"; -import { ExportDialog } from "./ExportDialog"; +import { ExportPage } from "./ExportPage"; import { Header } from "./Header"; import { PlayerContent } from "./Player"; import { Timeline } from "./Timeline"; -import { - Dialog, - DialogContent, - EditorButton, - Input, - Slider, - Subfield, -} from "./ui"; -import { formatTime } from "./utils"; +import { Dialog, DialogContent, EditorButton, Input, Subfield } from "./ui"; const DEFAULT_TIMELINE_HEIGHT = 260; const MIN_PLAYER_CONTENT_HEIGHT = 320; @@ -207,57 +200,66 @@ function Inner() { ), ); + const { dialog } = useEditorContext(); + + const isExportMode = () => { + const d = dialog(); + return "type" in d && d.type === "export" && d.open; + }; + return ( - <> -
-
+ }> +
+
-
- - - + ); } @@ -284,14 +286,11 @@ function Dialogs() { { const d = dialog(); - if ("type" in d) return d; + if ("type" in d && d.type !== "export") return d; })()} > {(dialog) => ( - - - {(_) => { const [form, setForm] = createStore({ @@ -422,59 +421,34 @@ function Dialogs() { })()} > {(dialog) => { - const { - setProject: setState, - editorInstance, - editorState, - totalDuration, - project, - } = useEditorContext(); + const { setProject: setState, editorInstance } = + useEditorContext(); const display = editorInstance.recordings.segments[0].display; let cropperRef: CropperRef | undefined; - let videoRef: HTMLVideoElement | undefined; const [crop, setCrop] = createSignal(CROP_ZERO); const [aspect, setAspect] = createSignal(null); - const [previewTime, setPreviewTime] = createSignal( - editorState.playbackTime, - ); - const [videoLoaded, setVideoLoaded] = createSignal(false); - - const currentSegment = createMemo(() => { - const time = previewTime(); - let elapsed = 0; - for (const seg of project.timeline?.segments ?? []) { - const segDuration = (seg.end - seg.start) / seg.timescale; - if (time < elapsed + segDuration) { - return { - index: seg.recordingSegment ?? 0, - localTime: seg.start / seg.timescale + (time - elapsed), - }; - } - elapsed += segDuration; - } - return { index: 0, localTime: 0 }; - }); - - const videoSrc = createMemo(() => - convertFileSrc( - `${editorInstance.path}/content/segments/segment-${currentSegment().index}/display.mp4`, - ), - ); - createEffect( - on( - () => currentSegment().index, - () => { - setVideoLoaded(false); - }, - { defer: true }, - ), - ); + const [frameBlobUrl, setFrameBlobUrl] = createSignal< + string | null + >(null); + + const playerCanvas = document.getElementById( + "canvas", + ) as HTMLCanvasElement | null; + if (playerCanvas) { + playerCanvas.toBlob((blob) => { + if (blob) { + const url = URL.createObjectURL(blob); + setFrameBlobUrl(url); + } + }, "image/png"); + } - createEffect(() => { - if (videoRef && videoLoaded()) { - videoRef.currentTime = currentSegment().localTime; + onCleanup(() => { + const url = frameBlobUrl(); + if (url) { + URL.revokeObjectURL(url); } }); @@ -638,60 +612,19 @@ function Dialogs() { allowLightMode={true} onContextMenu={(e) => showCropOptionsMenu(e, true)} > -
- screenshot -
+ ) + } + />
-
- - {formatTime(previewTime())} - - setPreviewTime(v)} - aria-label="Video timeline" - /> - - {formatTime(totalDuration())} - -
- ) - } - leftFooterContent={ -
- - - {(est) => ( -

- - - {(() => { - const totalSeconds = Math.round( - est().duration_seconds, - ); - const hours = Math.floor(totalSeconds / 3600); - const minutes = Math.floor( - (totalSeconds % 3600) / 60, - ); - const seconds = totalSeconds % 60; - - if (hours > 0) { - return `${hours}:${minutes - .toString() - .padStart(2, "0")}:${seconds - .toString() - .padStart(2, "0")}`; - } - return `${minutes}:${seconds - .toString() - .padStart(2, "0")}`; - })()} - - - - {settings.resolution.width}×{settings.resolution.height} - - - - {est().estimated_size_mb.toFixed(2)} MB - - - - {(() => { - const totalSeconds = Math.round( - est().estimated_time_seconds, - ); - const hours = Math.floor(totalSeconds / 3600); - const minutes = Math.floor( - (totalSeconds % 3600) / 60, - ); - const seconds = totalSeconds % 60; - - if (hours > 0) { - return `~${hours}:${minutes - .toString() - .padStart(2, "0")}:${seconds - .toString() - .padStart(2, "0")}`; - } - return `~${minutes}:${seconds - .toString() - .padStart(2, "0")}`; - })()} - -

- )} - - -
- } - > -
- {/* Export to */} -
-
-
-

Export to

- - 1 - } - > -
{ - const menu = await Menu.new({ - items: await Promise.all( - organisations().map((org) => - CheckMenuItem.new({ - text: org.name, - action: () => { - setSettings("organizationId", org.id); - }, - checked: settings.organizationId === org.id, - }), - ), - ), - }); - menu.popup(); - }} - > - Organization: - - { - ( - organisations().find( - (o) => o.id === settings.organizationId, - ) ?? organisations()[0] - )?.name - } - - -
- - -
-
- - {(option) => ( - - )} - -
-
-
- {/* Format */} -
-
-

Format

-
- - {(option) => { - const disabledReason = () => { - if ( - option.value === "Mp4" && - hasTransparentBackground() - ) - return "MP4 format does not support transparent backgrounds"; - if ( - option.value === "Gif" && - settings.exportTo === "link" - ) - return "Shareable links cannot be made from GIFs"; - }; - - return ( - - - - ); - }} - -
-
-
- {/* Frame rate */} -
-
-

Frame rate

- - options={ - settings.format === "Gif" ? GIF_FPS_OPTIONS : FPS_OPTIONS - } - optionValue="value" - optionTextValue="label" - placeholder="Select FPS" - value={(settings.format === "Gif" - ? GIF_FPS_OPTIONS - : FPS_OPTIONS - ).find((opt) => opt.value === settings.fps)} - onChange={(option) => { - if (!option) return; - trackEvent("export_fps_changed", { - fps: option.value, - }); - setSettings("fps", option.value); - }} - disallowEmptySelection - itemComponent={(props) => ( - - as={KSelect.Item} - item={props.item} - > - - {props.item.rawValue.label} - - - )} - > - - class="flex-1 text-sm text-left truncate tabular-nums text-[--gray-500]"> - {(state) => {state.selectedOption()?.label}} - - - as={(props) => ( - - )} - /> - - - - as={KSelect.Content} - class={cx(topSlideAnimateClasses, "z-50")} - > - - class="max-h-32 custom-scroll" - as={KSelect.Listbox} - /> - - - -
-
- {/* Compression */} -
-
-

Compression

-
- - {(option) => ( - - )} - -
-
-
- {/* Resolution */} -
-
-

Resolution

-
- - {(option) => ( - - )} - -
-
-
-
- - - - {(exportState) => { - const [copyPressed, setCopyPressed] = createSignal(false); - const [clipboardCopyPressed, setClipboardCopyPressed] = - createSignal(false); - const [showCompletionScreen, setShowCompletionScreen] = createSignal( - exportState.type === "done" && exportState.action === "save", - ); - - createEffect(() => { - if (exportState.type === "done" && exportState.action === "save") { - setShowCompletionScreen(true); - } - }); - - return ( - <> - -
- Export -
setDialog((d) => ({ ...d, open: false }))} - class="flex justify-center items-center p-1 rounded-full transition-colors cursor-pointer hover:bg-gray-3" - > - -
-
- - -
- - - {(copyState) => ( -
-

- {copyState.type === "starting" - ? "Preparing..." - : copyState.type === "rendering" - ? settings.format === "Gif" - ? "Rendering GIF..." - : "Rendering video..." - : copyState.type === "copying" - ? "Copying to clipboard..." - : "Copied to clipboard"} -

- - {(copyState) => ( - <> - - - - )} - -
- )} - - - {(saveState) => ( -
- -

- {saveState.type === "starting" - ? "Preparing..." - : saveState.type === "rendering" - ? settings.format === "Gif" - ? "Rendering GIF..." - : "Rendering video..." - : saveState.type === "copying" - ? "Exporting to file..." - : "Export completed"} -

- - {(copyState) => ( - <> - - - - )} - - - } - > -
-
-
- -
-
-

- Export Completed -

-

- Your{" "} - {settings.format === "Gif" - ? "GIF" - : "video"}{" "} - has successfully been exported -

-
-
-
- -
- )} - - - {(uploadState) => ( - - - {(uploadState) => ( -
-

- Uploading Cap... -

- - - {(uploadState) => ( - - )} - - - {(renderState) => ( - <> - - - - )} - - -
- )} - - -
-
-

- Upload Complete -

-

- Your Cap has been uploaded successfully -

-
-
- - - )} - - -
- - - - -
- - - -
- - - -
- - -
- - - - - ); - }} - - - ); -} - -function RenderProgress(props: { state: RenderState; format?: ExportFormat }) { - return ( - - ); -} - -function ProgressView(props: { amount: number; label?: string }) { - return ( - <> -
-
-
-

{props.label}

- - ); -} diff --git a/apps/desktop/src/routes/editor/ExportPage.tsx b/apps/desktop/src/routes/editor/ExportPage.tsx new file mode 100644 index 0000000000..a028e55976 --- /dev/null +++ b/apps/desktop/src/routes/editor/ExportPage.tsx @@ -0,0 +1,1442 @@ +import { Button } from "@cap/ui-solid"; +import { RadioGroup as KRadioGroup } from "@kobalte/core/radio-group"; +import { debounce } from "@solid-primitives/scheduled"; +import { makePersisted } from "@solid-primitives/storage"; +import { createMutation } from "@tanstack/solid-query"; +import { Channel } from "@tauri-apps/api/core"; +import { CheckMenuItem, Menu } from "@tauri-apps/api/menu"; +import { ask, save as saveDialog } from "@tauri-apps/plugin-dialog"; +import { remove } from "@tauri-apps/plugin-fs"; +import { type as ostype } from "@tauri-apps/plugin-os"; +import { cx } from "cva"; +import { + createEffect, + createSignal, + For, + type JSX, + Match, + mergeProps, + on, + onCleanup, + Show, + Suspense, + Switch, +} from "solid-js"; +import { createStore, produce, reconcile } from "solid-js/store"; +import toast from "solid-toast"; +import { SignInButton } from "~/components/SignInButton"; +import Tooltip from "~/components/Tooltip"; +import CaptionControlsWindows11 from "~/components/titlebar/controls/CaptionControlsWindows11"; +import { authStore } from "~/store"; +import { trackEvent } from "~/utils/analytics"; +import { createSignInMutation } from "~/utils/auth"; +import { createExportTask } from "~/utils/export"; +import { createOrganizationsQuery } from "~/utils/queries"; +import { + commands, + type ExportCompression, + type ExportSettings, + type FramesRendered, + type UploadProgress, +} from "~/utils/tauri"; +import { type RenderState, useEditorContext } from "./context"; +import { RESOLUTION_OPTIONS } from "./Header"; +import { Dialog, Field, Slider } from "./ui"; + +class SilentError extends Error {} + +export const COMPRESSION_OPTIONS: Array<{ + label: string; + value: ExportCompression; + bpp: number; +}> = [ + { label: "Minimal", value: "Minimal", bpp: 0.3 }, + { label: "Social Media", value: "Social", bpp: 0.15 }, + { label: "Web", value: "Web", bpp: 0.08 }, + { label: "Potato", value: "Potato", bpp: 0.04 }, +]; + +const BPP_TO_COMPRESSION: Record = { + "0.3": "Minimal", + "0.15": "Social", + "0.08": "Web", + "0.04": "Potato", +}; + +const COMPRESSION_TO_BPP: Record = { + Minimal: 0.3, + Social: 0.15, + Web: 0.08, + Potato: 0.04, +}; + +export const FPS_OPTIONS = [ + { label: "15 FPS", value: 15 }, + { label: "30 FPS", value: 30 }, + { label: "60 FPS", value: 60 }, +] satisfies Array<{ label: string; value: number }>; + +export const GIF_FPS_OPTIONS = [ + { label: "10 FPS", value: 10 }, + { label: "15 FPS", value: 15 }, + { label: "20 FPS", value: 20 }, + { label: "25 FPS", value: 25 }, + { label: "30 FPS", value: 30 }, +] satisfies Array<{ label: string; value: number }>; + +export const EXPORT_TO_OPTIONS = [ + { + label: "File", + value: "file", + icon: IconCapFile, + description: "Save to your computer", + }, + { + label: "Clipboard", + value: "clipboard", + icon: IconCapCopy, + description: "Copy to paste anywhere", + }, + { + label: "Shareable Link", + value: "link", + icon: IconCapLink, + description: "Share via Cap cloud", + }, +] as const; + +type ExportFormat = ExportSettings["format"]; + +const FORMAT_OPTIONS = [ + { label: "MP4", value: "Mp4" }, + { label: "GIF", value: "Gif" }, +] as { label: string; value: ExportFormat; disabled?: boolean }[]; + +type ExportToOption = (typeof EXPORT_TO_OPTIONS)[number]["value"]; + +interface Settings { + format: ExportFormat; + fps: number; + exportTo: ExportToOption; + resolution: { label: string; value: string; width: number; height: number }; + compression: ExportCompression; + organizationId?: string | null; +} + +export function ExportPage() { + const { + setDialog, + editorInstance, + editorState, + setExportState, + exportState, + meta, + refetchMeta, + } = useEditorContext(); + + const handleBack = () => { + setDialog((d) => ({ ...d, open: false })); + }; + + const projectPath = editorInstance.path; + + const auth = authStore.createQuery(); + const organisations = createOrganizationsQuery(); + + const hasTransparentBackground = () => { + const backgroundSource = + editorInstance.savedProjectConfig.background.source; + return ( + backgroundSource.type === "color" && + backgroundSource.alpha !== undefined && + backgroundSource.alpha < 255 + ); + }; + + const isCancellationError = (error: unknown) => + error instanceof SilentError || + error === "Export cancelled" || + (error instanceof Error && error.message === "Export cancelled"); + + const [_settings, setSettings] = makePersisted( + createStore({ + format: "Mp4", + fps: 30, + exportTo: "file", + resolution: { label: "720p", value: "720p", width: 1280, height: 720 }, + compression: "Minimal", + }), + { name: "export_settings" }, + ); + + const settings = mergeProps(_settings, () => { + const ret: Partial = {}; + if (hasTransparentBackground() && _settings.format === "Mp4") + ret.format = "Gif"; + else if (_settings.format === "Gif" && _settings.exportTo === "link") + ret.format = "Mp4"; + else if (!["Mp4", "Gif"].includes(_settings.format)) ret.format = "Mp4"; + + Object.defineProperty(ret, "organizationId", { + get() { + if (!_settings.organizationId && organisations().length > 0) + return organisations()[0].id; + + return _settings.organizationId; + }, + }); + + return ret; + }); + + const [previewUrl, setPreviewUrl] = createSignal(null); + const [previewLoading, setPreviewLoading] = createSignal(false); + const [renderEstimate, setRenderEstimate] = createSignal<{ + frameRenderTimeMs: number; + totalFrames: number; + estimatedSizeMb: number; + } | null>(null); + + type EstimateCacheKey = string; + const estimateCache = new Map< + EstimateCacheKey, + { frameRenderTimeMs: number; totalFrames: number; estimatedSizeMb: number } + >(); + + const getEstimateCacheKey = ( + fps: number, + width: number, + height: number, + bpp: number, + ): EstimateCacheKey => `${fps}-${width}-${height}-${bpp}`; + + const updateSettings: typeof setSettings = (( + ...args: Parameters + ) => { + setPreviewLoading(true); + return (setSettings as (...args: Parameters) => void)( + ...args, + ); + }) as typeof setSettings; + const [previewDialogOpen, setPreviewDialogOpen] = createSignal(false); + const [compressionBpp, setCompressionBpp] = createSignal( + COMPRESSION_TO_BPP[_settings.compression] ?? 0.15, + ); + + createEffect( + on( + () => _settings.compression, + (compression) => { + const bpp = COMPRESSION_TO_BPP[compression]; + if (bpp !== undefined) setCompressionBpp(bpp); + }, + ), + ); + + const debouncedFetchPreview = debounce( + async ( + frameTime: number, + fps: number, + resWidth: number, + resHeight: number, + bpp: number, + ) => { + const cacheKey = getEstimateCacheKey(fps, resWidth, resHeight, bpp); + const cachedEstimate = estimateCache.get(cacheKey); + + if (cachedEstimate) { + setRenderEstimate(cachedEstimate); + } + + try { + const result = await commands.generateExportPreviewFast(frameTime, { + fps, + resolution_base: { x: resWidth, y: resHeight }, + compression_bpp: bpp, + }); + + const oldUrl = previewUrl(); + if (oldUrl) URL.revokeObjectURL(oldUrl); + + const byteArray = Uint8Array.from(atob(result.jpeg_base64), (c) => + c.charCodeAt(0), + ); + const blob = new Blob([byteArray], { type: "image/jpeg" }); + setPreviewUrl(URL.createObjectURL(blob)); + + const newEstimate = { + frameRenderTimeMs: result.frame_render_time_ms, + totalFrames: result.total_frames, + estimatedSizeMb: result.estimated_size_mb, + }; + + if (!cachedEstimate) { + estimateCache.set(cacheKey, newEstimate); + } + setRenderEstimate(newEstimate); + } catch (e) { + console.error("Failed to generate preview:", e); + } finally { + setPreviewLoading(false); + } + }, + 300, + ); + + createEffect( + on( + [ + () => settings.format, + () => settings.fps, + () => settings.resolution.width, + () => settings.resolution.height, + compressionBpp, + ], + () => { + const frameTime = editorState.playbackTime ?? 0; + setPreviewLoading(true); + debouncedFetchPreview( + frameTime, + settings.fps, + settings.resolution.width, + settings.resolution.height, + compressionBpp(), + ); + }, + ), + ); + + onCleanup(() => { + const url = previewUrl(); + if (url) URL.revokeObjectURL(url); + }); + + let cancelCurrentExport: (() => void) | null = null; + + const exportWithSettings = ( + onProgress: (progress: FramesRendered) => void, + ) => { + const { promise, cancel } = createExportTask( + projectPath, + settings.format === "Mp4" + ? { + format: "Mp4", + fps: settings.fps, + resolution_base: { + x: settings.resolution.width, + y: settings.resolution.height, + }, + compression: settings.compression, + } + : { + format: "Gif", + fps: settings.fps, + resolution_base: { + x: settings.resolution.width, + y: settings.resolution.height, + }, + quality: null, + }, + onProgress, + ); + cancelCurrentExport = cancel; + return promise.finally(() => { + if (cancelCurrentExport === cancel) cancelCurrentExport = null; + }); + }; + + const [outputPath, setOutputPath] = createSignal(null); + const [isCancelled, setIsCancelled] = createSignal(false); + + const handleCancel = async () => { + if ( + await ask("Are you sure you want to cancel the export?", { + title: "Cancel Export", + kind: "warning", + }) + ) { + setIsCancelled(true); + cancelCurrentExport?.(); + cancelCurrentExport = null; + setExportState({ type: "idle" }); + const path = outputPath(); + if (path) { + try { + await remove(path); + } catch (e) { + console.error("Failed to delete cancelled file", e); + } + } + } + }; + + const copy = createMutation(() => ({ + mutationFn: async () => { + setIsCancelled(false); + if (exportState.type !== "idle") return; + setExportState(reconcile({ action: "copy", type: "starting" })); + + const outputPath = await exportWithSettings((progress) => { + if (isCancelled()) throw new SilentError("Cancelled"); + setExportState({ type: "rendering", progress }); + }); + + if (isCancelled()) throw new SilentError("Cancelled"); + + setExportState({ type: "copying" }); + + await commands.copyVideoToClipboard(outputPath); + }, + onError: (error) => { + if (isCancelled() || isCancellationError(error)) { + setExportState(reconcile({ type: "idle" })); + return; + } + commands.globalMessageDialog( + error instanceof Error ? error.message : "Failed to copy recording", + ); + setExportState(reconcile({ type: "idle" })); + }, + onSuccess() { + setExportState({ type: "done" }); + toast.success( + `${ + settings.format === "Gif" ? "GIF" : "Recording" + } exported to clipboard`, + ); + }, + })); + + const save = createMutation(() => ({ + mutationFn: async () => { + setIsCancelled(false); + if (exportState.type !== "idle") return; + + const extension = settings.format === "Gif" ? "gif" : "mp4"; + const savePath = await saveDialog({ + filters: [ + { + name: `${extension.toUpperCase()} filter`, + extensions: [extension], + }, + ], + defaultPath: `~/Desktop/${meta().prettyName}.${extension}`, + }); + if (!savePath) { + setExportState(reconcile({ type: "idle" })); + return; + } + + setExportState(reconcile({ action: "save", type: "starting" })); + + setOutputPath(savePath); + + trackEvent("export_started", { + resolution: settings.resolution, + fps: settings.fps, + path: savePath, + }); + + const videoPath = await exportWithSettings((progress) => { + if (isCancelled()) throw new SilentError("Cancelled"); + setExportState({ type: "rendering", progress }); + }); + + if (isCancelled()) throw new SilentError("Cancelled"); + + setExportState({ type: "copying" }); + + await commands.copyFileToPath(videoPath, savePath); + + setExportState({ type: "done" }); + }, + onError: (error) => { + if (isCancelled() || isCancellationError(error)) { + setExportState({ type: "idle" }); + return; + } + commands.globalMessageDialog( + error instanceof Error + ? error.message + : `Failed to export recording: ${error}`, + ); + setExportState({ type: "idle" }); + }, + onSuccess() { + toast.success( + `${settings.format === "Gif" ? "GIF" : "Recording"} exported to file`, + ); + }, + })); + + const upload = createMutation(() => ({ + mutationFn: async () => { + setIsCancelled(false); + if (exportState.type !== "idle") return; + setExportState(reconcile({ action: "upload", type: "starting" })); + + const existingAuth = await authStore.get(); + if (!existingAuth) createSignInMutation(); + trackEvent("create_shareable_link_clicked", { + resolution: settings.resolution, + fps: settings.fps, + has_existing_auth: !!existingAuth, + }); + + const metadata = await commands.getVideoMetadata(projectPath); + const plan = await commands.checkUpgradedAndUpdate(); + const canShare = { + allowed: plan || metadata.duration < 300, + reason: !plan && metadata.duration >= 300 ? "upgrade_required" : null, + }; + + if (!canShare.allowed) { + if (canShare.reason === "upgrade_required") { + await commands.showWindow("Upgrade"); + await new Promise((resolve) => setTimeout(resolve, 1000)); + throw new SilentError(); + } + } + + const uploadChannel = new Channel((progress) => { + console.log("Upload progress:", progress); + setExportState( + produce((state) => { + if (state.type !== "uploading") return; + + state.progress = Math.round(progress.progress * 100); + }), + ); + }); + + await exportWithSettings((progress) => { + if (isCancelled()) throw new SilentError("Cancelled"); + setExportState({ type: "rendering", progress }); + }); + + if (isCancelled()) throw new SilentError("Cancelled"); + + setExportState({ type: "uploading", progress: 0 }); + + console.log({ organizationId: settings.organizationId }); + + const result = meta().sharing + ? await commands.uploadExportedVideo( + projectPath, + "Reupload", + uploadChannel, + settings.organizationId ?? null, + ) + : await commands.uploadExportedVideo( + projectPath, + { Initial: { pre_created_video: null } }, + uploadChannel, + settings.organizationId ?? null, + ); + + if (result === "NotAuthenticated") + throw new Error("You need to sign in to share recordings"); + else if (result === "PlanCheckFailed") + throw new Error("Failed to verify your subscription status"); + else if (result === "UpgradeRequired") + throw new Error("This feature requires an upgraded plan"); + }, + onSuccess: async () => { + await refetchMeta(); + setExportState({ type: "done" }); + }, + onError: (error) => { + if (isCancelled() || isCancellationError(error)) { + setExportState(reconcile({ type: "idle" })); + return; + } + console.error(error); + if (!(error instanceof SilentError)) { + commands.globalMessageDialog( + error instanceof Error ? error.message : "Failed to upload recording", + ); + } + + setExportState(reconcile({ type: "idle" })); + }, + })); + + const qualityLabel = () => { + const option = COMPRESSION_OPTIONS.find( + (opt) => opt.value === settings.compression, + ); + return option?.label ?? "Minimal"; + }; + + const formatDuration = (seconds: number) => { + const hours = Math.floor(seconds / 3600); + const minutes = Math.floor((seconds % 3600) / 60); + const secs = seconds % 60; + if (hours > 0) { + return `${hours}:${minutes.toString().padStart(2, "0")}:${secs.toString().padStart(2, "0")}`; + } + return `${minutes}:${secs.toString().padStart(2, "0")}`; + }; + + return ( +
+
+

+ Export +

+
+ {ostype() === "macos" &&
} + +
+ {ostype() === "windows" && } +
+
+ +
+
+
+ Preview + + + +
+
+ + + + Generating preview... +
+ } + > +
+
+
+ +
+ } + > + {(url) => ( + <> + Export preview + +
+
+
+ + + + )} + +
+ + + + + + + + + + + + + + + + + + +
+ } + > + {(est) => { + const data = est(); + const durationSeconds = data.totalFrames / settings.fps; + + const exportSpeedMultiplier = settings.format === "Gif" ? 4 : 10; + const totalTimeMs = + (data.frameRenderTimeMs * data.totalFrames) / + exportSpeedMultiplier; + const estimatedTimeSeconds = Math.max(1, totalTimeMs / 1000); + + const sizeMultiplier = settings.format === "Gif" ? 0.7 : 0.5; + const estimatedSizeMb = data.estimatedSizeMb * sizeMultiplier; + + return ( +
+ + + + {formatDuration(Math.round(durationSeconds))} + + + + + + {settings.resolution.width}×{settings.resolution.height} + + + + + + ~{estimatedSizeMb.toFixed(1)} MB + + + + + + ~{formatDuration(Math.round(estimatedTimeSeconds))} + + +
+ ); + }} + +
+ +
+
+ }> + { + setSettings( + produce((newSettings) => { + newSettings.exportTo = value as ExportToOption; + if (value === "link" && settings.format === "Gif") { + newSettings.format = "Mp4"; + } + }), + ); + }} + > + + {(option) => { + const Icon = option.icon; + return ( + + + + + +
+ + {option.label} + + + {option.description} + +
+ + + ); + }} + + + + + 1 + } + > + + + + + + }> +
+ + {(option) => { + const isDisabled = () => + (option.value === "Mp4" && hasTransparentBackground()) || + (option.value === "Gif" && settings.exportTo === "link"); + + const disabledReason = () => + option.value === "Mp4" && hasTransparentBackground() + ? "MP4 doesn't support transparency" + : option.value === "Gif" && settings.exportTo === "link" + ? "Links require MP4 format" + : undefined; + + const button = ( + + ); + + return disabledReason() ? ( + {button} + ) : ( + button + ); + }} + +
+ + + } + > +
+ + {(option) => ( + + )} + +
+ + + } + value={ + + {settings.fps} FPS + + } + > +
+ + {(option) => ( + + )} + +
+ + + + } + value={ + {qualityLabel()} + } + > + opt.value === settings.compression, + ), + ]} + minValue={0} + maxValue={COMPRESSION_OPTIONS.length - 1} + step={1} + onChange={([v]) => { + if (v === undefined) return; + const option = + COMPRESSION_OPTIONS[COMPRESSION_OPTIONS.length - 1 - v]; + if (option) { + setPreviewLoading(true); + setCompressionBpp(option.bpp); + setSettings("compression", option.value); + } + }} + history={{ pause: () => () => {} }} + /> + + +
+ +
+ {settings.exportTo === "link" && !auth.data ? ( + + + Sign in to share + + ) : ( + + )} +
+
+
+ + +
+
+

Quality Preview

+ +
+
+ + {(url) => ( + Export preview full size + )} + +
+
+ + {settings.resolution.width}×{settings.resolution.height} + + + {(est) => { + const sizeMultiplier = settings.format === "Gif" ? 0.7 : 0.5; + return ( + + Estimated size:{" "} + {(est().estimatedSizeMb * sizeMultiplier).toFixed(1)} MB + + ); + }} + +
+
+ + + + {(exportState) => { + const [copyPressed, setCopyPressed] = createSignal(false); + const [clipboardCopyPressed, setClipboardCopyPressed] = + createSignal(false); + const [showCompletionScreen, setShowCompletionScreen] = createSignal( + exportState.type === "done" && exportState.action === "save", + ); + + createEffect(() => { + if (exportState.type === "done" && exportState.action === "save") { + setShowCompletionScreen(true); + } + }); + + return ( +
+
+ + + {(copyState) => ( +
+

+ {copyState.type === "starting" + ? "Preparing..." + : copyState.type === "rendering" + ? settings.format === "Gif" + ? "Rendering GIF..." + : "Rendering video..." + : copyState.type === "copying" + ? "Copying to clipboard..." + : "Copied to clipboard"} +

+ + {(copyState) => ( + <> + + + + )} + +
+ )} + + + {(saveState) => ( +
+ +

+ {saveState.type === "starting" + ? "Preparing..." + : saveState.type === "rendering" + ? settings.format === "Gif" + ? "Rendering GIF..." + : "Rendering video..." + : saveState.type === "copying" + ? "Exporting to file..." + : "Export completed"} +

+ + {(copyState) => ( + <> + + + + )} + + + } + > +
+
+
+ +
+
+

+ Export Complete +

+

+ Your{" "} + {settings.format === "Gif" ? "GIF" : "video"}{" "} + is ready +

+
+
+
+ +
+ )} + + + {(uploadState) => ( + + + {(uploadState) => ( +
+

+ {uploadState.type === "uploading" + ? "Uploading..." + : "Preparing..."} +

+ + + {(uploadState) => ( + + )} + + + {(renderState) => ( + <> + + + + )} + + +
+ )} + + +
+
+

+ Upload Complete +

+

+ Your Cap has been uploaded successfully +

+
+
+ + + )} + + +
+ +
+ + + {(link) => ( +
+ + + + +
+ )} + + + + +
+ + +
+ +
+ + + + +
+ ); + }} + +
+ ); +} + +function RenderProgress(props: { state: RenderState; format?: ExportFormat }) { + return ( + + ); +} + +function ProgressView(props: { amount: number; label?: string }) { + return ( + <> +
+
+
+

{props.label}

+ + ); +} diff --git a/apps/desktop/src/routes/editor/Header.tsx b/apps/desktop/src/routes/editor/Header.tsx index 86f7ecba70..905813d14c 100644 --- a/apps/desktop/src/routes/editor/Header.tsx +++ b/apps/desktop/src/routes/editor/Header.tsx @@ -165,7 +165,7 @@ export function Header() { {ostype() === "windows" && } diff --git a/apps/desktop/src/routes/editor/editor-skeleton.tsx b/apps/desktop/src/routes/editor/editor-skeleton.tsx new file mode 100644 index 0000000000..372bbfdfcf --- /dev/null +++ b/apps/desktop/src/routes/editor/editor-skeleton.tsx @@ -0,0 +1,232 @@ +import { type as ostype } from "@tauri-apps/plugin-os"; +import { cx } from "cva"; + +const DEFAULT_TIMELINE_HEIGHT = 260; +const MIN_PLAYER_HEIGHT = 328; +const RESIZE_HANDLE_HEIGHT = 8; + +function SkeletonPulse(props: { class?: string }) { + return ( +
+ ); +} + +function SkeletonButton(props: { class?: string; width?: string }) { + return ( + + ); +} + +function HeaderSkeleton() { + return ( +
+
+ {ostype() === "macos" &&
} + + + + +
+ + +
+ +
+ +
+ +
+ + +
+ +
+
+ ); +} + +function PlayerToolbarSkeleton() { + return ( +
+
+ + +
+
+ + +
+
+ ); +} + +function VideoPreviewSkeleton() { + return ( +
+
+
+
+ +
+
+
+
+ ); +} + +function PlayerControlsSkeleton() { + return ( +
+
+ + + +
+
+ + + +
+
+
+ + + + + +
+
+ ); +} + +function PlayerSkeleton() { + return ( +
+
+ + + +
+
+
+
+
+
+
+ ); +} + +function SidebarSkeleton() { + return ( +
+
+ + + + + + +
+
+ +
+ + + + +
+ + +
+ + + + + + + + +
+
+
+ ); +} + +function TimelineTrackSkeleton() { + return ( +
+
+ +
+
+ +
+
+ ); +} + +function TimelineSkeleton() { + return ( +
+
+
+
+ + + + + +
+
+ +
+
+
+ + +
+
+
+ ); +} + +export function EditorSkeleton() { + return ( + <> + +
+
+
+ + +
+
+
+ +
+
+
+
+ + ); +} diff --git a/apps/desktop/src/routes/editor/index.tsx b/apps/desktop/src/routes/editor/index.tsx index 1d45af3caf..ca0bfdf6f6 100644 --- a/apps/desktop/src/routes/editor/index.tsx +++ b/apps/desktop/src/routes/editor/index.tsx @@ -2,10 +2,10 @@ import { Effect, getCurrentWindow } from "@tauri-apps/api/window"; import { type as ostype } from "@tauri-apps/plugin-os"; import { cx } from "cva"; import { createEffect, Suspense } from "solid-js"; -import { AbsoluteInsetLoader } from "~/components/Loader"; import { generalSettingsStore } from "~/store"; import { commands } from "~/utils/tauri"; import { Editor } from "./Editor"; +import { EditorSkeleton } from "./editor-skeleton"; export default function () { const generalSettings = generalSettingsStore.createQuery(); @@ -27,7 +27,7 @@ export default function () { ) && "bg-transparent-window", )} > - }> + }>
diff --git a/apps/desktop/src/utils/tauri.ts b/apps/desktop/src/utils/tauri.ts index 77c8a8a94e..18ba0fbe66 100644 --- a/apps/desktop/src/utils/tauri.ts +++ b/apps/desktop/src/utils/tauri.ts @@ -86,6 +86,12 @@ async exportVideo(projectPath: string, progress: TAURI_CHANNEL, async getExportEstimates(path: string, settings: ExportSettings) : Promise { return await TAURI_INVOKE("get_export_estimates", { path, settings }); }, +async generateExportPreview(projectPath: string, frameTime: number, settings: ExportPreviewSettings) : Promise { + return await TAURI_INVOKE("generate_export_preview", { projectPath, frameTime, settings }); +}, +async generateExportPreviewFast(frameTime: number, settings: ExportPreviewSettings) : Promise { + return await TAURI_INVOKE("generate_export_preview_fast", { frameTime, settings }); +}, async copyFileToPath(src: string, dst: string) : Promise { return await TAURI_INVOKE("copy_file_to_path", { src, dst }); }, @@ -408,6 +414,8 @@ export type DownloadProgress = { progress: number; message: string } export type EditorStateChanged = { playhead_position: number } export type ExportCompression = "Minimal" | "Social" | "Web" | "Potato" export type ExportEstimates = { duration_seconds: number; estimated_time_seconds: number; estimated_size_mb: number } +export type ExportPreviewResult = { jpeg_base64: string; estimated_size_mb: number; actual_width: number; actual_height: number; frame_render_time_ms: number; total_frames: number } +export type ExportPreviewSettings = { fps: number; resolution_base: XY; compression_bpp: number } export type ExportSettings = ({ format: "Mp4" } & Mp4ExportSettings) | ({ format: "Gif" } & GifExportSettings) export type FileType = "recording" | "screenshot" export type Flags = { captions: boolean } diff --git a/apps/desktop/tailwind.config.js b/apps/desktop/tailwind.config.js index 8fd2b2602a..6ab4fa0e40 100644 --- a/apps/desktop/tailwind.config.js +++ b/apps/desktop/tailwind.config.js @@ -13,10 +13,54 @@ module.exports = { "0%, 100%": { transform: "translateY(0)" }, "50%": { transform: "translateY(-4px)" }, }, + shimmer: { + "0%": { transform: "translateX(-100%)" }, + "100%": { transform: "translateX(100%)" }, + }, + float: { + "0%, 100%": { + transform: "translateY(0) rotate(0deg)", + opacity: "0.7", + }, + "50%": { + transform: "translateY(-20px) rotate(180deg)", + opacity: "1", + }, + }, + floatSlow: { + "0%, 100%": { transform: "translateY(0) scale(1)" }, + "50%": { transform: "translateY(-30px) scale(1.1)" }, + }, + pulse3d: { + "0%, 100%": { transform: "scale(1)", opacity: "0.8" }, + "50%": { transform: "scale(1.05)", opacity: "1" }, + }, + spin3d: { + "0%": { transform: "rotateY(0deg)" }, + "100%": { transform: "rotateY(360deg)" }, + }, + gradientShift: { + "0%": { backgroundPosition: "0% 50%" }, + "50%": { backgroundPosition: "100% 50%" }, + "100%": { backgroundPosition: "0% 50%" }, + }, + dash: { + "0%": { strokeDasharray: "1, 150", strokeDashoffset: "0" }, + "50%": { strokeDasharray: "90, 150", strokeDashoffset: "-35" }, + "100%": { strokeDasharray: "90, 150", strokeDashoffset: "-124" }, + }, }, animation: { ...baseConfig.theme?.extend?.animation, "gentle-bounce": "gentleBounce 1.5s ease-in-out infinite", + shimmer: "shimmer 2s ease-in-out infinite", + float: "float 6s ease-in-out infinite", + "float-slow": "floatSlow 8s ease-in-out infinite", + "float-delayed": "float 6s ease-in-out 2s infinite", + pulse3d: "pulse3d 2s ease-in-out infinite", + spin3d: "spin3d 3s linear infinite", + "gradient-shift": "gradientShift 3s ease infinite", + dash: "dash 1.5s ease-in-out infinite", }, }, }, diff --git a/crates/editor/Cargo.toml b/crates/editor/Cargo.toml index 4410cd6fac..c612d1e33f 100644 --- a/crates/editor/Cargo.toml +++ b/crates/editor/Cargo.toml @@ -6,6 +6,10 @@ edition = "2024" [lints] workspace = true +[[example]] +name = "decode-benchmark" +path = "examples/decode-benchmark.rs" + [dependencies] cap-media = { path = "../media" } cap-project = { path = "../project" } diff --git a/crates/editor/examples/decode-benchmark.rs b/crates/editor/examples/decode-benchmark.rs new file mode 100644 index 0000000000..c71c18b6be --- /dev/null +++ b/crates/editor/examples/decode-benchmark.rs @@ -0,0 +1,419 @@ +use cap_rendering::decoder::{AsyncVideoDecoderHandle, spawn_decoder}; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::time::Instant; +use tokio::runtime::Runtime; + +const DEFAULT_DURATION_SECS: f32 = 60.0; + +fn get_video_duration(path: &Path) -> f32 { + let output = Command::new("ffprobe") + .args([ + "-v", + "error", + "-show_entries", + "format=duration", + "-of", + "default=noprint_wrappers=1:nokey=1", + ]) + .arg(path) + .output(); + + match output { + Ok(output) if output.status.success() => { + let duration_str = String::from_utf8_lossy(&output.stdout); + duration_str.trim().parse().unwrap_or(DEFAULT_DURATION_SECS) + } + _ => { + eprintln!( + "Warning: Could not determine video duration via ffprobe, using default {DEFAULT_DURATION_SECS}s" + ); + DEFAULT_DURATION_SECS + } + } +} + +#[derive(Debug, Clone)] +struct BenchmarkConfig { + video_path: PathBuf, + fps: u32, + iterations: usize, +} + +#[derive(Debug, Default)] +struct BenchmarkResults { + decoder_creation_ms: f64, + sequential_decode_times_ms: Vec, + sequential_fps: f64, + sequential_failures: usize, + seek_times_by_distance: Vec<(f32, f64)>, + seek_failures: usize, + random_access_times_ms: Vec, + random_access_avg_ms: f64, + random_access_failures: usize, + cache_hits: usize, + cache_misses: usize, +} + +impl BenchmarkResults { + fn print_report(&self) { + println!("\n{}", "=".repeat(60)); + println!(" VIDEO DECODE BENCHMARK RESULTS"); + println!("{}\n", "=".repeat(60)); + + println!("DECODER CREATION"); + println!( + " Time to create decoder: {:.2}ms", + self.decoder_creation_ms + ); + println!(); + + println!("SEQUENTIAL DECODE PERFORMANCE"); + if !self.sequential_decode_times_ms.is_empty() || self.sequential_failures > 0 { + let avg: f64 = if self.sequential_decode_times_ms.is_empty() { + 0.0 + } else { + self.sequential_decode_times_ms.iter().sum::() + / self.sequential_decode_times_ms.len() as f64 + }; + let min = self + .sequential_decode_times_ms + .iter() + .cloned() + .fold(f64::INFINITY, f64::min); + let max = self + .sequential_decode_times_ms + .iter() + .cloned() + .fold(f64::NEG_INFINITY, f64::max); + println!( + " Frames decoded: {}", + self.sequential_decode_times_ms.len() + ); + if self.sequential_failures > 0 { + println!(" Frames failed: {}", self.sequential_failures); + } + println!(" Avg decode time: {avg:.2}ms"); + println!(" Min decode time: {min:.2}ms"); + println!(" Max decode time: {max:.2}ms"); + println!(" Effective FPS: {:.1}", self.sequential_fps); + } + println!(); + + println!("SEEK PERFORMANCE (by distance)"); + if !self.seek_times_by_distance.is_empty() || self.seek_failures > 0 { + println!(" {:>10} | {:>12}", "Distance(s)", "Time(ms)"); + println!(" {}-+-{}", "-".repeat(10), "-".repeat(12)); + for (distance, time) in &self.seek_times_by_distance { + println!(" {distance:>10.1} | {time:>12.2}"); + } + if self.seek_failures > 0 { + println!(" Seek failures: {}", self.seek_failures); + } + } + println!(); + + println!("RANDOM ACCESS PERFORMANCE"); + if !self.random_access_times_ms.is_empty() || self.random_access_failures > 0 { + let avg = if self.random_access_times_ms.is_empty() { + 0.0 + } else { + self.random_access_times_ms.iter().sum::() + / self.random_access_times_ms.len() as f64 + }; + let min = self + .random_access_times_ms + .iter() + .copied() + .fold(f64::INFINITY, f64::min); + let max = self + .random_access_times_ms + .iter() + .copied() + .fold(f64::NEG_INFINITY, f64::max); + println!(" Samples: {}", self.random_access_times_ms.len()); + if self.random_access_failures > 0 { + println!(" Failures: {}", self.random_access_failures); + } + println!(" Avg access time: {avg:.2}ms"); + println!(" Min access time: {min:.2}ms"); + println!(" Max access time: {max:.2}ms"); + println!( + " P50: {:.2}ms", + percentile(&self.random_access_times_ms, 50.0) + ); + println!( + " P95: {:.2}ms", + percentile(&self.random_access_times_ms, 95.0) + ); + println!( + " P99: {:.2}ms", + percentile(&self.random_access_times_ms, 99.0) + ); + } + println!(); + + let total = self.cache_hits + self.cache_misses; + if total > 0 { + println!("CACHE STATISTICS"); + println!( + " Hits: {} ({:.1}%)", + self.cache_hits, + 100.0 * self.cache_hits as f64 / total as f64 + ); + println!( + " Misses: {} ({:.1}%)", + self.cache_misses, + 100.0 * self.cache_misses as f64 / total as f64 + ); + } + + println!("\n{}\n", "=".repeat(60)); + } +} + +fn percentile(data: &[f64], p: f64) -> f64 { + let filtered: Vec = data.iter().copied().filter(|x| !x.is_nan()).collect(); + if filtered.is_empty() { + return 0.0; + } + let mut sorted = filtered; + sorted.sort_by(|a, b| { + a.partial_cmp(b) + .expect("NaN values should have been filtered out") + }); + let idx = ((p / 100.0) * (sorted.len() - 1) as f64).round() as usize; + sorted[idx.min(sorted.len() - 1)] +} + +async fn benchmark_decoder_creation(path: &Path, fps: u32, iterations: usize) -> f64 { + let mut total_ms = 0.0; + + for i in 0..iterations { + let start = Instant::now(); + let decoder = spawn_decoder("benchmark", path.to_path_buf(), fps, 0.0).await; + let elapsed = start.elapsed(); + + match decoder { + Ok(_) => { + total_ms += elapsed.as_secs_f64() * 1000.0; + } + Err(e) => { + if i == 0 { + eprintln!("Failed to create decoder: {e}"); + return -1.0; + } + } + } + } + + total_ms / iterations as f64 +} + +async fn benchmark_sequential_decode( + decoder: &AsyncVideoDecoderHandle, + fps: u32, + frame_count: usize, + start_time: f32, +) -> (Vec, f64, usize) { + let mut times = Vec::with_capacity(frame_count); + let mut failures = 0; + let overall_start = Instant::now(); + + for i in 0..frame_count { + let time = start_time + (i as f32 / fps as f32); + let start = Instant::now(); + match decoder.get_frame(time).await { + Some(_frame) => { + let elapsed = start.elapsed(); + times.push(elapsed.as_secs_f64() * 1000.0); + } + None => { + failures += 1; + eprintln!("Failed to get frame at time {time:.3}s"); + } + } + } + + let overall_elapsed = overall_start.elapsed(); + let successful_frames = frame_count - failures; + let effective_fps = if overall_elapsed.as_secs_f64() > 0.0 { + successful_frames as f64 / overall_elapsed.as_secs_f64() + } else { + 0.0 + }; + + (times, effective_fps, failures) +} + +async fn benchmark_seek( + decoder: &AsyncVideoDecoderHandle, + _fps: u32, + from_time: f32, + to_time: f32, +) -> Option { + if decoder.get_frame(from_time).await.is_none() { + eprintln!("Failed to get initial frame at time {from_time:.3}s for seek benchmark"); + return None; + } + + let start = Instant::now(); + match decoder.get_frame(to_time).await { + Some(_frame) => { + let elapsed = start.elapsed(); + Some(elapsed.as_secs_f64() * 1000.0) + } + None => { + eprintln!("Failed to get frame at time {to_time:.3}s for seek benchmark"); + None + } + } +} + +async fn benchmark_random_access( + decoder: &AsyncVideoDecoderHandle, + _fps: u32, + duration_secs: f32, + sample_count: usize, +) -> (Vec, usize) { + let mut times = Vec::with_capacity(sample_count); + let mut failures = 0; + + let golden_ratio = 1.618_034_f32; + let mut position = 0.0_f32; + + for _ in 0..sample_count { + position = (position + golden_ratio * duration_secs) % duration_secs; + let start = Instant::now(); + match decoder.get_frame(position).await { + Some(_frame) => { + let elapsed = start.elapsed(); + times.push(elapsed.as_secs_f64() * 1000.0); + } + None => { + failures += 1; + eprintln!("Failed to get frame at position {position:.3}s during random access"); + } + } + } + + (times, failures) +} + +async fn run_full_benchmark(config: BenchmarkConfig) -> BenchmarkResults { + let mut results = BenchmarkResults::default(); + + println!( + "Starting benchmark with video: {}", + config.video_path.display() + ); + println!("FPS: {}, Iterations: {}", config.fps, config.iterations); + println!(); + + println!("[1/5] Benchmarking decoder creation..."); + results.decoder_creation_ms = + benchmark_decoder_creation(&config.video_path, config.fps, config.iterations).await; + if results.decoder_creation_ms < 0.0 { + eprintln!("Failed to benchmark decoder creation"); + return results; + } + println!(" Done: {:.2}ms avg", results.decoder_creation_ms); + + println!("[2/5] Creating decoder for remaining tests..."); + let decoder = match spawn_decoder("benchmark", config.video_path.clone(), config.fps, 0.0).await + { + Ok(d) => d, + Err(e) => { + eprintln!("Failed to create decoder: {e}"); + return results; + } + }; + println!(" Done"); + + let video_duration = get_video_duration(&config.video_path); + println!("Detected video duration: {video_duration:.2}s"); + println!(); + + println!("[3/5] Benchmarking sequential decode (100 frames from start)..."); + let (seq_times, seq_fps, seq_failures) = + benchmark_sequential_decode(&decoder, config.fps, 100, 0.0).await; + results.sequential_decode_times_ms = seq_times; + results.sequential_fps = seq_fps; + results.sequential_failures = seq_failures; + println!(" Done: {seq_fps:.1} effective FPS"); + if seq_failures > 0 { + println!(" Warning: {seq_failures} frames failed to decode"); + } + + println!("[4/5] Benchmarking seek performance..."); + let seek_distances: Vec = vec![0.5, 1.0, 2.0, 5.0, 10.0, 30.0] + .into_iter() + .filter(|&d| d <= video_duration) + .collect(); + for distance in seek_distances { + match benchmark_seek(&decoder, config.fps, 0.0, distance).await { + Some(seek_time) => { + results.seek_times_by_distance.push((distance, seek_time)); + println!(" {distance:.1}s seek: {seek_time:.2}ms"); + } + None => { + results.seek_failures += 1; + println!(" {distance:.1}s seek: FAILED"); + } + } + } + + println!("[5/5] Benchmarking random access (50 samples)..."); + let (random_times, random_failures) = + benchmark_random_access(&decoder, config.fps, video_duration, 50).await; + results.random_access_times_ms = random_times; + results.random_access_failures = random_failures; + results.random_access_avg_ms = if results.random_access_times_ms.is_empty() { + 0.0 + } else { + results.random_access_times_ms.iter().sum::() + / results.random_access_times_ms.len() as f64 + }; + println!(" Done: {:.2}ms avg", results.random_access_avg_ms); + if random_failures > 0 { + println!(" Warning: {random_failures} random accesses failed"); + } + + results +} + +fn main() { + let args: Vec = std::env::args().collect(); + + let video_path = args + .iter() + .position(|a| a == "--video") + .and_then(|i| args.get(i + 1)) + .map(PathBuf::from) + .expect("Usage: decode-benchmark --video [--fps ] [--iterations ]"); + + let fps = args + .iter() + .position(|a| a == "--fps") + .and_then(|i| args.get(i + 1)) + .and_then(|s| s.parse().ok()) + .unwrap_or(30); + + let iterations = args + .iter() + .position(|a| a == "--iterations") + .and_then(|i| args.get(i + 1)) + .and_then(|s| s.parse().ok()) + .unwrap_or(100); + + let config = BenchmarkConfig { + video_path, + fps, + iterations, + }; + + let rt = Runtime::new().expect("Failed to create Tokio runtime"); + let results = rt.block_on(run_full_benchmark(config)); + + results.print_report(); +} diff --git a/crates/editor/src/audio.rs b/crates/editor/src/audio.rs index 0e1032bb75..215018ff09 100644 --- a/crates/editor/src/audio.rs +++ b/crates/editor/src/audio.rs @@ -277,6 +277,10 @@ impl AudioPlaybackBuffer { self.frame_buffer.set_playhead(playhead, project); } + pub fn current_playhead(&self) -> f64 { + self.frame_buffer.elapsed_samples_to_playhead() + } + pub fn buffer_reaching_limit(&self) -> bool { self.resampled_buffer.vacant_len() <= 2 * (Self::PROCESSING_SAMPLES_COUNT as usize) * self.resampler.output.channels diff --git a/crates/editor/src/playback.rs b/crates/editor/src/playback.rs index f02469dcca..4096abfb1b 100644 --- a/crates/editor/src/playback.rs +++ b/crates/editor/src/playback.rs @@ -30,11 +30,11 @@ use crate::{ segments::get_audio_segments, }; -const PREFETCH_BUFFER_SIZE: usize = 180; -const PARALLEL_DECODE_TASKS: usize = 20; -const MAX_PREFETCH_AHEAD: u32 = 240; -const PREFETCH_BEHIND: u32 = 60; -const FRAME_CACHE_SIZE: usize = 150; +const PREFETCH_BUFFER_SIZE: usize = 60; +const PARALLEL_DECODE_TASKS: usize = 8; +const MAX_PREFETCH_AHEAD: u32 = 90; +const PREFETCH_BEHIND: u32 = 15; +const FRAME_CACHE_SIZE: usize = 60; #[derive(Debug)] pub enum PlaybackStartError { @@ -333,12 +333,16 @@ impl Playback { f64::MAX }; + let (audio_playhead_tx, audio_playhead_rx) = + watch::channel(self.start_frame_number as f64 / fps as f64); + AudioPlayback { segments: get_audio_segments(&self.segment_medias), stop_rx: stop_rx.clone(), start_frame_number: self.start_frame_number, project: self.project.clone(), fps, + playhead_rx: audio_playhead_rx, } .spawn(); @@ -352,8 +356,8 @@ impl Playback { let mut total_frames_rendered = 0u64; let mut _total_frames_skipped = 0u64; - let warmup_target_frames = 2usize; - let warmup_after_first_timeout = Duration::from_millis(50); + let warmup_target_frames = 1usize; + let warmup_after_first_timeout = Duration::from_millis(16); let mut first_frame_time: Option = None; while !*stop_rx.borrow() { @@ -627,6 +631,12 @@ impl Playback { frame_number = frame_number.saturating_add(1); let _ = playback_position_tx.send(frame_number); + if audio_playhead_tx + .send(frame_number as f64 / fps_f64) + .is_err() + { + break 'playback; + } let expected_frame = self.start_frame_number + (start.elapsed().as_secs_f64() * fps_f64).floor() as u32; @@ -646,6 +656,12 @@ impl Playback { prefetch_buffer.retain(|p| p.frame_number >= frame_number); let _ = frame_request_tx.send(frame_number); let _ = playback_position_tx.send(frame_number); + if audio_playhead_tx + .send(frame_number as f64 / fps_f64) + .is_err() + { + break 'playback; + } } } } @@ -676,6 +692,7 @@ struct AudioPlayback { start_frame_number: u32, project: watch::Receiver, fps: u32, + playhead_rx: watch::Receiver, } impl AudioPlayback { @@ -761,7 +778,7 @@ impl AudioPlayback { project, segments, fps, - .. + playhead_rx, } = self; let mut base_output_info = AudioInfo::from_stream_config(&supported_config); @@ -915,6 +932,9 @@ impl AudioPlayback { let project_for_stream = project.clone(); let headroom_for_stream = headroom_samples; + let mut playhead_rx_for_stream = playhead_rx.clone(); + let mut last_video_playhead = playhead; + const SYNC_THRESHOLD_SECS: f64 = 0.15; let stream_result = device.build_output_stream( &config, @@ -923,6 +943,20 @@ impl AudioPlayback { let project = project_for_stream.borrow(); + if playhead_rx_for_stream.has_changed().unwrap_or(false) { + let video_playhead = *playhead_rx_for_stream.borrow_and_update(); + let audio_playhead = audio_renderer.current_playhead(); + let drift = (video_playhead - audio_playhead).abs(); + + if drift > SYNC_THRESHOLD_SECS + || (video_playhead - last_video_playhead).abs() > SYNC_THRESHOLD_SECS + { + audio_renderer + .set_playhead(video_playhead + initial_compensation_secs, &project); + } + last_video_playhead = video_playhead; + } + let playback_samples = buffer.len(); let min_headroom = headroom_for_stream.max(playback_samples * 2); audio_renderer.fill(buffer, &project, min_headroom); diff --git a/crates/enc-avfoundation/src/lib.rs b/crates/enc-avfoundation/src/lib.rs index 8683aafff5..1db0f16db5 100644 --- a/crates/enc-avfoundation/src/lib.rs +++ b/crates/enc-avfoundation/src/lib.rs @@ -1,7 +1,5 @@ #![cfg(target_os = "macos")] mod mp4; -mod segmented; pub use mp4::*; -pub use segmented::*; diff --git a/crates/enc-avfoundation/src/mp4.rs b/crates/enc-avfoundation/src/mp4.rs index e0be1560cc..2498624be0 100644 --- a/crates/enc-avfoundation/src/mp4.rs +++ b/crates/enc-avfoundation/src/mp4.rs @@ -79,6 +79,15 @@ impl MP4Encoder { audio_config: Option, output_height: Option, ) -> Result { + info!( + width = video_config.width, + height = video_config.height, + pixel_format = ?video_config.pixel_format, + frame_rate = ?video_config.frame_rate, + output_height = ?output_height, + has_audio = audio_config.is_some(), + "Initializing AVFoundation MP4 encoder (VideoToolbox hardware encoding)" + ); debug!("{video_config:#?}"); debug!("{audio_config:#?}"); @@ -122,11 +131,23 @@ impl MP4Encoder { debug!("recording bitrate: {bitrate}"); + let keyframe_interval = (fps * 2.0) as i32; + output_settings.insert( av::video_settings_keys::compression_props(), ns::Dictionary::with_keys_values( - &[unsafe { AVVideoAverageBitRateKey }], - &[ns::Number::with_f32(bitrate).as_id_ref()], + &[ + unsafe { AVVideoAverageBitRateKey }, + unsafe { AVVideoAllowFrameReorderingKey }, + unsafe { AVVideoExpectedSourceFrameRateKey }, + unsafe { AVVideoMaxKeyFrameIntervalKey }, + ], + &[ + ns::Number::with_f32(bitrate).as_id_ref(), + ns::Number::with_bool(false).as_id_ref(), + ns::Number::with_f32(fps).as_id_ref(), + ns::Number::with_i32(keyframe_interval).as_id_ref(), + ], ) .as_id_ref(), ); @@ -524,6 +545,9 @@ impl Drop for MP4Encoder { #[link(name = "AVFoundation", kind = "framework")] unsafe extern "C" { static AVVideoAverageBitRateKey: &'static ns::String; + static AVVideoAllowFrameReorderingKey: &'static ns::String; + static AVVideoExpectedSourceFrameRateKey: &'static ns::String; + static AVVideoMaxKeyFrameIntervalKey: &'static ns::String; static AVVideoTransferFunctionKey: &'static ns::String; static AVVideoColorPrimariesKey: &'static ns::String; static AVVideoYCbCrMatrixKey: &'static ns::String; diff --git a/crates/enc-avfoundation/src/segmented.rs b/crates/enc-avfoundation/src/segmented.rs deleted file mode 100644 index 1c0cff38af..0000000000 --- a/crates/enc-avfoundation/src/segmented.rs +++ /dev/null @@ -1,429 +0,0 @@ -use crate::{FinishError, InitError, MP4Encoder, QueueFrameError, wait_for_writer_finished}; -use cap_media_info::{AudioInfo, VideoInfo}; -use cidre::arc; -use ffmpeg::frame; -use serde::Serialize; -use std::{ - io::Write, - path::{Path, PathBuf}, - time::Duration, -}; -use tracing::warn; - -fn atomic_write_json(path: &Path, data: &T) -> std::io::Result<()> { - let temp_path = path.with_extension("json.tmp"); - let json = serde_json::to_string_pretty(data) - .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; - - let mut file = std::fs::File::create(&temp_path)?; - file.write_all(json.as_bytes())?; - file.sync_all()?; - - std::fs::rename(&temp_path, path)?; - - if let Some(parent) = path.parent() - && let Ok(dir) = std::fs::File::open(parent) - { - let _ = dir.sync_all(); - } - - Ok(()) -} - -fn sync_file(path: &Path) { - if let Ok(file) = std::fs::File::open(path) { - let _ = file.sync_all(); - } -} - -pub struct SegmentedMP4Encoder { - base_path: PathBuf, - video_config: VideoInfo, - audio_config: Option, - output_height: Option, - - current_encoder: Option, - current_index: u32, - segment_duration: Duration, - segment_start_time: Option, - - completed_segments: Vec, -} - -#[derive(Debug, Clone)] -pub struct SegmentInfo { - pub path: PathBuf, - pub index: u32, - pub duration: Duration, - pub file_size: Option, - pub is_failed: bool, -} - -#[derive(Serialize)] -struct FragmentEntry { - path: String, - index: u32, - duration: f64, - is_complete: bool, - #[serde(skip_serializing_if = "Option::is_none")] - file_size: Option, - #[serde(skip_serializing_if = "std::ops::Not::not")] - is_failed: bool, -} - -const MANIFEST_VERSION: u32 = 2; - -#[derive(Serialize)] -struct Manifest { - version: u32, - fragments: Vec, - #[serde(skip_serializing_if = "Option::is_none")] - total_duration: Option, - is_complete: bool, -} - -impl SegmentedMP4Encoder { - pub fn init( - base_path: PathBuf, - video_config: VideoInfo, - audio_config: Option, - output_height: Option, - segment_duration: Duration, - ) -> Result { - std::fs::create_dir_all(&base_path).map_err(|_| InitError::NoSettingsAssistant)?; - - let segment_path = base_path.join("fragment_000.mp4"); - let encoder = MP4Encoder::init(segment_path, video_config, audio_config, output_height)?; - - let instance = Self { - base_path, - video_config, - audio_config, - output_height, - current_encoder: Some(encoder), - current_index: 0, - segment_duration, - segment_start_time: None, - completed_segments: Vec::new(), - }; - - instance.write_in_progress_manifest(); - - Ok(instance) - } - - pub fn queue_video_frame( - &mut self, - frame: arc::R, - timestamp: Duration, - ) -> Result<(), QueueFrameError> { - if self.segment_start_time.is_none() { - self.segment_start_time = Some(timestamp); - } - - let segment_elapsed = - timestamp.saturating_sub(self.segment_start_time.unwrap_or(Duration::ZERO)); - - if segment_elapsed >= self.segment_duration { - self.rotate_segment(timestamp)?; - } - - if let Some(encoder) = &mut self.current_encoder { - encoder.queue_video_frame(frame, timestamp) - } else { - Err(QueueFrameError::NoEncoder) - } - } - - pub fn queue_audio_frame( - &mut self, - frame: &frame::Audio, - timestamp: Duration, - ) -> Result<(), QueueFrameError> { - if let Some(encoder) = &mut self.current_encoder { - encoder.queue_audio_frame(frame, timestamp) - } else { - Err(QueueFrameError::NoEncoder) - } - } - - fn rotate_segment(&mut self, timestamp: Duration) -> Result<(), QueueFrameError> { - let segment_start = self.segment_start_time.unwrap_or(Duration::ZERO); - let segment_duration = timestamp.saturating_sub(segment_start); - let completed_segment_path = self.current_segment_path(); - let current_index = self.current_index; - - if let Some(mut encoder) = self.current_encoder.take() { - let finish_failed = match encoder.finish_nowait(Some(timestamp)) { - Ok(writer) => { - let path_for_sync = completed_segment_path.clone(); - std::thread::spawn(move || { - if let Err(e) = wait_for_writer_finished(&writer) { - warn!( - "Background writer finalization failed for segment {current_index}: {e}" - ); - } - sync_file(&path_for_sync); - }); - false - } - Err(e) => { - tracing::error!( - "Failed to finish encoder during rotation for segment {}: {e}", - current_index - ); - true - } - }; - - let file_size = std::fs::metadata(&completed_segment_path) - .ok() - .map(|m| m.len()); - - self.completed_segments.push(SegmentInfo { - path: completed_segment_path, - index: current_index, - duration: segment_duration, - file_size, - is_failed: finish_failed, - }); - - self.write_manifest(); - - if finish_failed { - tracing::warn!( - "Segment {} marked as failed in manifest, continuing with new segment", - current_index - ); - } - } - - self.current_index += 1; - self.segment_start_time = Some(timestamp); - - let new_path = self.current_segment_path(); - self.current_encoder = Some( - MP4Encoder::init( - new_path, - self.video_config, - self.audio_config, - self.output_height, - ) - .map_err(|e| { - tracing::error!( - "Failed to create new encoder for segment {}: {e}", - self.current_index - ); - QueueFrameError::Failed - })?, - ); - - self.write_in_progress_manifest(); - - Ok(()) - } - - fn current_segment_path(&self) -> PathBuf { - self.base_path - .join(format!("fragment_{:03}.mp4", self.current_index)) - } - - fn write_manifest(&self) { - let manifest = Manifest { - version: MANIFEST_VERSION, - fragments: self - .completed_segments - .iter() - .map(|s| FragmentEntry { - path: s - .path - .file_name() - .unwrap_or_default() - .to_string_lossy() - .into_owned(), - index: s.index, - duration: s.duration.as_secs_f64(), - is_complete: !s.is_failed, - file_size: s.file_size, - is_failed: s.is_failed, - }) - .collect(), - total_duration: None, - is_complete: false, - }; - - let manifest_path = self.base_path.join("manifest.json"); - if let Err(e) = atomic_write_json(&manifest_path, &manifest) { - tracing::warn!( - "Failed to write manifest to {}: {e}", - manifest_path.display() - ); - } - } - - fn write_in_progress_manifest(&self) { - let mut fragments: Vec = self - .completed_segments - .iter() - .map(|s| FragmentEntry { - path: s - .path - .file_name() - .unwrap_or_default() - .to_string_lossy() - .into_owned(), - index: s.index, - duration: s.duration.as_secs_f64(), - is_complete: !s.is_failed, - file_size: s.file_size, - is_failed: s.is_failed, - }) - .collect(); - - fragments.push(FragmentEntry { - path: self - .current_segment_path() - .file_name() - .unwrap_or_default() - .to_string_lossy() - .into_owned(), - index: self.current_index, - duration: 0.0, - is_complete: false, - file_size: None, - is_failed: false, - }); - - let manifest = Manifest { - version: MANIFEST_VERSION, - fragments, - total_duration: None, - is_complete: false, - }; - - let manifest_path = self.base_path.join("manifest.json"); - if let Err(e) = atomic_write_json(&manifest_path, &manifest) { - tracing::warn!( - "Failed to write in-progress manifest to {}: {e}", - manifest_path.display() - ); - } - } - - pub fn pause(&mut self) { - if let Some(encoder) = &mut self.current_encoder { - encoder.pause(); - } - } - - pub fn resume(&mut self) { - if let Some(encoder) = &mut self.current_encoder { - encoder.resume(); - } - } - - pub fn finish(&mut self, timestamp: Option) -> Result<(), FinishError> { - let segment_path = self.current_segment_path(); - let segment_start = self.segment_start_time; - let current_index = self.current_index; - - if let Some(mut encoder) = self.current_encoder.take() { - match encoder.finish_nowait(timestamp) { - Ok(writer) => { - let path_for_sync = segment_path.clone(); - std::thread::spawn(move || { - if let Err(e) = wait_for_writer_finished(&writer) { - warn!( - "Background writer finalization failed for segment {current_index}: {e}" - ); - } - sync_file(&path_for_sync); - }); - - if let Some(start) = segment_start { - let final_duration = timestamp.unwrap_or(start).saturating_sub(start); - let file_size = std::fs::metadata(&segment_path).ok().map(|m| m.len()); - - self.completed_segments.push(SegmentInfo { - path: segment_path, - index: current_index, - duration: final_duration, - file_size, - is_failed: false, - }); - } - } - Err(e) => { - tracing::error!("Failed to finish final segment {current_index}: {e}"); - - if let Some(start) = segment_start { - let final_duration = timestamp.unwrap_or(start).saturating_sub(start); - let file_size = std::fs::metadata(&segment_path).ok().map(|m| m.len()); - - self.completed_segments.push(SegmentInfo { - path: segment_path, - index: current_index, - duration: final_duration, - file_size, - is_failed: true, - }); - } - } - } - } - - self.finalize_manifest(); - - Ok(()) - } - - fn finalize_manifest(&self) { - let total_duration: Duration = self.completed_segments.iter().map(|s| s.duration).sum(); - let has_failed_segments = self.completed_segments.iter().any(|s| s.is_failed); - - if has_failed_segments { - tracing::warn!( - "Recording completed with {} failed segment(s)", - self.completed_segments - .iter() - .filter(|s| s.is_failed) - .count() - ); - } - - let manifest = Manifest { - version: MANIFEST_VERSION, - fragments: self - .completed_segments - .iter() - .map(|s| FragmentEntry { - path: s - .path - .file_name() - .unwrap_or_default() - .to_string_lossy() - .into_owned(), - index: s.index, - duration: s.duration.as_secs_f64(), - is_complete: !s.is_failed, - file_size: s.file_size, - is_failed: s.is_failed, - }) - .collect(), - total_duration: Some(total_duration.as_secs_f64()), - is_complete: true, - }; - - let manifest_path = self.base_path.join("manifest.json"); - if let Err(e) = atomic_write_json(&manifest_path, &manifest) { - tracing::warn!( - "Failed to write final manifest to {}: {e}", - manifest_path.display() - ); - } - } - - pub fn completed_segments(&self) -> &[SegmentInfo] { - &self.completed_segments - } -} diff --git a/crates/enc-ffmpeg/src/lib.rs b/crates/enc-ffmpeg/src/lib.rs index d57097346c..2dd64f560d 100644 --- a/crates/enc-ffmpeg/src/lib.rs +++ b/crates/enc-ffmpeg/src/lib.rs @@ -13,6 +13,3 @@ pub mod remux; pub mod segmented_audio { pub use crate::mux::segmented_audio::*; } -pub mod fragmented_mp4 { - pub use crate::mux::fragmented_mp4::*; -} diff --git a/crates/enc-ffmpeg/src/mux/fragmented_mp4.rs b/crates/enc-ffmpeg/src/mux/fragmented_mp4.rs deleted file mode 100644 index 20f1b034ef..0000000000 --- a/crates/enc-ffmpeg/src/mux/fragmented_mp4.rs +++ /dev/null @@ -1,170 +0,0 @@ -use cap_media_info::RawVideoFormat; -use ffmpeg::{format, frame}; -use std::{path::PathBuf, time::Duration}; -use tracing::*; - -use crate::{ - audio::AudioEncoder, - h264, - video::h264::{H264Encoder, H264EncoderError}, -}; - -pub struct FragmentedMP4File { - output: format::context::Output, - video: H264Encoder, - audio: Option>, - is_finished: bool, - has_frames: bool, -} - -#[derive(thiserror::Error, Debug)] -pub enum InitError { - #[error("{0:?}")] - Ffmpeg(ffmpeg::Error), - #[error("Video/{0}")] - VideoInit(H264EncoderError), - #[error("Audio/{0}")] - AudioInit(Box), - #[error("Failed to create output directory: {0}")] - CreateDirectory(std::io::Error), -} - -#[derive(thiserror::Error, Debug)] -pub enum FinishError { - #[error("Already finished")] - AlreadyFinished, - #[error("{0}")] - WriteTrailerFailed(ffmpeg::Error), -} - -pub struct FinishResult { - pub video_finish: Result<(), ffmpeg::Error>, - pub audio_finish: Result<(), ffmpeg::Error>, -} - -impl FragmentedMP4File { - pub fn init( - mut output_path: PathBuf, - video: impl FnOnce(&mut format::context::Output) -> Result, - audio: impl FnOnce( - &mut format::context::Output, - ) - -> Option, Box>>, - ) -> Result { - output_path.set_extension("mp4"); - - if let Some(parent) = output_path.parent() { - std::fs::create_dir_all(parent).map_err(InitError::CreateDirectory)?; - } - - let mut output = format::output_as(&output_path, "mp4").map_err(InitError::Ffmpeg)?; - - unsafe { - let opts = output.as_mut_ptr(); - let key = std::ffi::CString::new("movflags").unwrap(); - let value = - std::ffi::CString::new("frag_keyframe+empty_moov+default_base_moof").unwrap(); - ffmpeg::ffi::av_opt_set((*opts).priv_data, key.as_ptr(), value.as_ptr(), 0); - } - - trace!("Preparing encoders for fragmented mp4 file"); - - let video = video(&mut output).map_err(InitError::VideoInit)?; - let audio = audio(&mut output) - .transpose() - .map_err(InitError::AudioInit)?; - - info!("Prepared encoders for fragmented mp4 file"); - - output.write_header().map_err(InitError::Ffmpeg)?; - - Ok(Self { - output, - video, - audio, - is_finished: false, - has_frames: false, - }) - } - - pub fn video_format() -> RawVideoFormat { - RawVideoFormat::Yuv420p - } - - pub fn queue_video_frame( - &mut self, - frame: frame::Video, - timestamp: Duration, - ) -> Result<(), h264::QueueFrameError> { - if self.is_finished { - return Ok(()); - } - - self.has_frames = true; - self.video.queue_frame(frame, timestamp, &mut self.output) - } - - pub fn queue_audio_frame(&mut self, frame: frame::Audio) { - if self.is_finished { - return; - } - - let Some(audio) = &mut self.audio else { - return; - }; - - self.has_frames = true; - audio.send_frame(frame, &mut self.output); - } - - pub fn finish(&mut self) -> Result { - if self.is_finished { - return Err(FinishError::AlreadyFinished); - } - - self.is_finished = true; - - tracing::info!("FragmentedMP4File: Finishing encoding"); - - let video_finish = self.video.flush(&mut self.output).inspect_err(|e| { - error!("Failed to finish video encoder: {e:#}"); - }); - - let audio_finish = self - .audio - .as_mut() - .map(|enc| { - tracing::info!("FragmentedMP4File: Flushing audio encoder"); - enc.flush(&mut self.output).inspect_err(|e| { - error!("Failed to finish audio encoder: {e:#}"); - }) - }) - .unwrap_or(Ok(())); - - tracing::info!("FragmentedMP4File: Writing trailer"); - self.output - .write_trailer() - .map_err(FinishError::WriteTrailerFailed)?; - - Ok(FinishResult { - video_finish, - audio_finish, - }) - } - - pub fn video(&self) -> &H264Encoder { - &self.video - } - - pub fn video_mut(&mut self) -> &mut H264Encoder { - &mut self.video - } -} - -impl Drop for FragmentedMP4File { - fn drop(&mut self) { - if let Err(e) = self.finish() { - error!("Failed to finish FragmentedMP4File in Drop: {e}"); - } - } -} diff --git a/crates/enc-ffmpeg/src/mux/mod.rs b/crates/enc-ffmpeg/src/mux/mod.rs index 8ed8362d1b..28b1fcf0db 100644 --- a/crates/enc-ffmpeg/src/mux/mod.rs +++ b/crates/enc-ffmpeg/src/mux/mod.rs @@ -1,5 +1,5 @@ pub mod fragmented_audio; -pub mod fragmented_mp4; pub mod mp4; pub mod ogg; pub mod segmented_audio; +pub mod segmented_stream; diff --git a/crates/enc-ffmpeg/src/mux/segmented_audio.rs b/crates/enc-ffmpeg/src/mux/segmented_audio.rs index 1f54d3e743..e24a8ff058 100644 --- a/crates/enc-ffmpeg/src/mux/segmented_audio.rs +++ b/crates/enc-ffmpeg/src/mux/segmented_audio.rs @@ -367,6 +367,10 @@ impl SegmentedAudioEncoder { pub fn finish_with_timestamp(&mut self, timestamp: Duration) -> Result<(), FinishError> { let segment_path = self.current_segment_path(); let segment_start = self.segment_start_time; + let effective_end_timestamp = self + .last_frame_timestamp + .map(|last| last.max(timestamp)) + .unwrap_or(timestamp); if let Some(mut encoder) = self.current_encoder.take() { if encoder.has_frames { @@ -383,7 +387,7 @@ impl SegmentedAudioEncoder { sync_file(&segment_path); if let Some(start) = segment_start { - let final_duration = timestamp.saturating_sub(start); + let final_duration = effective_end_timestamp.saturating_sub(start); let file_size = std::fs::metadata(&segment_path).ok().map(|m| m.len()); self.completed_segments.push(SegmentInfo { diff --git a/crates/enc-ffmpeg/src/mux/segmented_stream.rs b/crates/enc-ffmpeg/src/mux/segmented_stream.rs new file mode 100644 index 0000000000..91df680cc2 --- /dev/null +++ b/crates/enc-ffmpeg/src/mux/segmented_stream.rs @@ -0,0 +1,599 @@ +use cap_media_info::VideoInfo; +use ffmpeg::{format, frame}; +use serde::Serialize; +use std::{ + ffi::CString, + io::Write, + path::{Path, PathBuf}, + time::Duration, +}; + +use crate::video::h264::{H264Encoder, H264EncoderBuilder, H264EncoderError, H264Preset}; + +const INIT_SEGMENT_NAME: &str = "init.mp4"; + +fn atomic_write_json(path: &Path, data: &T) -> std::io::Result<()> { + let temp_path = path.with_extension("json.tmp"); + let json = serde_json::to_string_pretty(data) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; + + let mut file = std::fs::File::create(&temp_path)?; + file.write_all(json.as_bytes())?; + file.sync_all()?; + + std::fs::rename(&temp_path, path)?; + + if let Some(parent) = path.parent() + && let Ok(dir) = std::fs::File::open(parent) + && let Err(e) = dir.sync_all() + { + tracing::warn!( + "Directory fsync failed after rename for {}: {e}", + parent.display() + ); + } + + Ok(()) +} + +fn sync_file(path: &Path) { + if let Ok(file) = std::fs::File::open(path) + && let Err(e) = file.sync_all() + { + tracing::warn!("File fsync failed for {}: {e}", path.display()); + } +} + +pub struct SegmentedVideoEncoder { + base_path: PathBuf, + + encoder: H264Encoder, + output: format::context::Output, + + current_index: u32, + segment_duration: Duration, + segment_start_time: Option, + last_frame_timestamp: Option, + frames_in_segment: u32, + + completed_segments: Vec, +} + +#[derive(Debug, Clone)] +pub struct VideoSegmentInfo { + pub path: PathBuf, + pub index: u32, + pub duration: Duration, + pub file_size: Option, +} + +#[derive(Serialize)] +struct SegmentEntry { + path: String, + index: u32, + duration: f64, + is_complete: bool, + #[serde(skip_serializing_if = "Option::is_none")] + file_size: Option, +} + +const MANIFEST_VERSION: u32 = 4; + +#[derive(Serialize)] +struct Manifest { + version: u32, + #[serde(rename = "type")] + manifest_type: &'static str, + #[serde(skip_serializing_if = "Option::is_none")] + init_segment: Option, + segments: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + total_duration: Option, + is_complete: bool, +} + +#[derive(thiserror::Error, Debug)] +pub enum InitError { + #[error("FFmpeg: {0}")] + FFmpeg(#[from] ffmpeg::Error), + #[error("Encoder: {0}")] + Encoder(#[from] H264EncoderError), + #[error("IO: {0}")] + Io(#[from] std::io::Error), +} + +#[derive(thiserror::Error, Debug)] +pub enum QueueFrameError { + #[error("FFmpeg: {0}")] + FFmpeg(#[from] ffmpeg::Error), + #[error("Init: {0}")] + Init(#[from] InitError), + #[error("Encode: {0}")] + Encode(#[from] crate::video::h264::QueueFrameError), +} + +#[derive(thiserror::Error, Debug)] +pub enum FinishError { + #[error("FFmpeg: {0}")] + FFmpeg(#[from] ffmpeg::Error), +} + +pub struct SegmentedVideoEncoderConfig { + pub segment_duration: Duration, + pub preset: H264Preset, + pub bpp: f32, + pub output_size: Option<(u32, u32)>, +} + +impl Default for SegmentedVideoEncoderConfig { + fn default() -> Self { + Self { + segment_duration: Duration::from_secs(3), + preset: H264Preset::Ultrafast, + bpp: H264EncoderBuilder::QUALITY_BPP, + output_size: None, + } + } +} + +impl SegmentedVideoEncoder { + pub fn init( + base_path: PathBuf, + video_config: VideoInfo, + config: SegmentedVideoEncoderConfig, + ) -> Result { + std::fs::create_dir_all(&base_path)?; + + let manifest_path = base_path.join("dash_manifest.mpd"); + + let mut output = format::output_as(&manifest_path, "dash")?; + + unsafe { + let opts = output.as_mut_ptr(); + + let set_opt = |key: &str, value: &str| { + let k = CString::new(key).unwrap(); + let v = CString::new(value).unwrap(); + ffmpeg::ffi::av_opt_set((*opts).priv_data, k.as_ptr(), v.as_ptr(), 0); + }; + + set_opt("init_seg_name", INIT_SEGMENT_NAME); + set_opt("media_seg_name", "segment_$Number%03d$.m4s"); + set_opt( + "seg_duration", + &config.segment_duration.as_secs_f64().to_string(), + ); + set_opt("use_timeline", "0"); + set_opt("use_template", "1"); + set_opt("single_file", "0"); + } + + let mut builder = H264EncoderBuilder::new(video_config) + .with_preset(config.preset) + .with_bpp(config.bpp); + + if let Some((width, height)) = config.output_size { + builder = builder.with_output_size(width, height)?; + } + + let encoder = builder.build(&mut output)?; + + output.write_header()?; + + tracing::info!( + path = %base_path.display(), + segment_duration_secs = config.segment_duration.as_secs(), + "Initialized segmented video encoder with FFmpeg DASH muxer (init.mp4 + m4s segments)" + ); + + let instance = Self { + base_path, + encoder, + output, + current_index: 1, + segment_duration: config.segment_duration, + segment_start_time: None, + last_frame_timestamp: None, + frames_in_segment: 0, + completed_segments: Vec::new(), + }; + + instance.write_in_progress_manifest(); + + Ok(instance) + } + + pub fn queue_frame( + &mut self, + frame: frame::Video, + timestamp: Duration, + ) -> Result<(), QueueFrameError> { + if self.segment_start_time.is_none() { + self.segment_start_time = Some(timestamp); + } + + self.last_frame_timestamp = Some(timestamp); + + let prev_segment_index = self.detect_current_segment_index(); + + self.encoder + .queue_frame(frame, timestamp, &mut self.output)?; + self.frames_in_segment += 1; + + let new_segment_index = self.detect_current_segment_index(); + + if new_segment_index > prev_segment_index { + self.on_segment_completed(prev_segment_index, timestamp)?; + } + + Ok(()) + } + + fn detect_current_segment_index(&self) -> u32 { + let next_segment_path = self + .base_path + .join(format!("segment_{:03}.m4s", self.current_index + 1)); + if next_segment_path.exists() { + self.current_index + 1 + } else { + self.current_index + } + } + + fn on_segment_completed( + &mut self, + completed_index: u32, + timestamp: Duration, + ) -> Result<(), QueueFrameError> { + let segment_path = self + .base_path + .join(format!("segment_{completed_index:03}.m4s")); + + if segment_path.exists() { + sync_file(&segment_path); + + let segment_start = self.segment_start_time.unwrap_or(Duration::ZERO); + let segment_duration = timestamp.saturating_sub(segment_start); + + let file_size = std::fs::metadata(&segment_path).ok().map(|m| m.len()); + + tracing::debug!( + segment_index = completed_index, + duration_secs = segment_duration.as_secs_f64(), + file_size = ?file_size, + frames = self.frames_in_segment, + "Segment completed" + ); + + self.completed_segments.push(VideoSegmentInfo { + path: segment_path, + index: completed_index, + duration: segment_duration, + file_size, + }); + + self.current_index = completed_index + 1; + self.segment_start_time = Some(timestamp); + self.frames_in_segment = 0; + + self.write_manifest(); + self.write_in_progress_manifest(); + } + + Ok(()) + } + + fn current_segment_path(&self) -> PathBuf { + self.base_path + .join(format!("segment_{:03}.m4s", self.current_index)) + } + + fn write_manifest(&self) { + let manifest = Manifest { + version: MANIFEST_VERSION, + manifest_type: "m4s_segments", + init_segment: Some(INIT_SEGMENT_NAME.to_string()), + segments: self + .completed_segments + .iter() + .map(|s| SegmentEntry { + path: s + .path + .file_name() + .unwrap_or_default() + .to_string_lossy() + .into_owned(), + index: s.index, + duration: s.duration.as_secs_f64(), + is_complete: true, + file_size: s.file_size, + }) + .collect(), + total_duration: None, + is_complete: false, + }; + + let manifest_path = self.base_path.join("manifest.json"); + if let Err(e) = atomic_write_json(&manifest_path, &manifest) { + tracing::warn!( + "Failed to write manifest to {}: {e}", + manifest_path.display() + ); + } + } + + fn write_in_progress_manifest(&self) { + let mut segments: Vec = self + .completed_segments + .iter() + .map(|s| SegmentEntry { + path: s + .path + .file_name() + .unwrap_or_default() + .to_string_lossy() + .into_owned(), + index: s.index, + duration: s.duration.as_secs_f64(), + is_complete: true, + file_size: s.file_size, + }) + .collect(); + + segments.push(SegmentEntry { + path: self + .current_segment_path() + .file_name() + .unwrap_or_default() + .to_string_lossy() + .into_owned(), + index: self.current_index, + duration: 0.0, + is_complete: false, + file_size: None, + }); + + let manifest = Manifest { + version: MANIFEST_VERSION, + manifest_type: "m4s_segments", + init_segment: Some(INIT_SEGMENT_NAME.to_string()), + segments, + total_duration: None, + is_complete: false, + }; + + let manifest_path = self.base_path.join("manifest.json"); + if let Err(e) = atomic_write_json(&manifest_path, &manifest) { + tracing::warn!( + "Failed to write in-progress manifest to {}: {e}", + manifest_path.display() + ); + } + } + + pub fn finish(&mut self) -> Result<(), FinishError> { + let segment_start = self.segment_start_time; + let last_timestamp = self.last_frame_timestamp; + let frames_before_flush = self.frames_in_segment; + + if let Err(e) = self.encoder.flush(&mut self.output) { + tracing::warn!("Video encoder flush warning: {e}"); + } + + if let Err(e) = self.output.write_trailer() { + tracing::warn!("Video write_trailer warning: {e}"); + } + + self.finalize_pending_tmp_files(); + + let end_timestamp = + last_timestamp.unwrap_or_else(|| segment_start.unwrap_or(Duration::ZERO)); + self.collect_orphaned_segments(segment_start, end_timestamp, frames_before_flush); + + self.finalize_manifest(); + + Ok(()) + } + + pub fn finish_with_timestamp(&mut self, timestamp: Duration) -> Result<(), FinishError> { + let segment_start = self.segment_start_time; + let frames_before_flush = self.frames_in_segment; + + if let Err(e) = self.encoder.flush(&mut self.output) { + tracing::warn!("Video encoder flush warning: {e}"); + } + + if let Err(e) = self.output.write_trailer() { + tracing::warn!("Video write_trailer warning: {e}"); + } + + self.finalize_pending_tmp_files(); + + let effective_end_timestamp = self + .last_frame_timestamp + .map(|last| last.max(timestamp)) + .unwrap_or(timestamp); + + self.collect_orphaned_segments(segment_start, effective_end_timestamp, frames_before_flush); + + self.finalize_manifest(); + + Ok(()) + } + + fn finalize_pending_tmp_files(&self) { + let Ok(entries) = std::fs::read_dir(&self.base_path) else { + return; + }; + + for entry in entries.flatten() { + let path = entry.path(); + if let Some(name) = path.file_name().and_then(|n| n.to_str()) + && name.starts_with("segment_") + && name.ends_with(".m4s.tmp") + && let Ok(metadata) = std::fs::metadata(&path) + && metadata.len() > 0 + { + let final_name = name.trim_end_matches(".tmp"); + let final_path = self.base_path.join(final_name); + + if let Err(e) = std::fs::rename(&path, &final_path) { + tracing::warn!( + "Failed to rename tmp segment {} to {}: {}", + path.display(), + final_path.display(), + e + ); + } else { + tracing::debug!( + "Finalized pending segment: {} ({} bytes)", + final_path.display(), + metadata.len() + ); + sync_file(&final_path); + } + } + } + } + + fn collect_orphaned_segments( + &mut self, + segment_start: Option, + end_timestamp: Duration, + frames_before_flush: u32, + ) { + let completed_indices: std::collections::HashSet = + self.completed_segments.iter().map(|s| s.index).collect(); + + let Ok(entries) = std::fs::read_dir(&self.base_path) else { + return; + }; + + let mut orphaned: Vec<(u32, PathBuf)> = Vec::new(); + + for entry in entries.flatten() { + let path = entry.path(); + if let Some(name) = path.file_name().and_then(|n| n.to_str()) + && name.starts_with("segment_") + && name.ends_with(".m4s") + && !name.contains(".tmp") + && let Some(index_str) = name + .strip_prefix("segment_") + .and_then(|s| s.strip_suffix(".m4s")) + && let Ok(index) = index_str.parse::() + && !completed_indices.contains(&index) + { + orphaned.push((index, path)); + } + } + + orphaned.sort_by_key(|(idx, _)| *idx); + + for (index, segment_path) in orphaned { + if let Ok(metadata) = std::fs::metadata(&segment_path) { + let file_size = metadata.len(); + + if file_size < 100 { + tracing::debug!( + "Skipping tiny orphaned segment {} ({} bytes)", + segment_path.display(), + file_size + ); + continue; + } + + sync_file(&segment_path); + + let duration = if index == self.current_index && frames_before_flush > 0 { + if let Some(start) = segment_start { + end_timestamp.saturating_sub(start) + } else { + self.segment_duration + } + } else { + self.segment_duration + }; + + tracing::info!( + "Recovered orphaned segment {} with {} bytes, estimated duration {:?}", + segment_path.display(), + file_size, + duration + ); + + self.completed_segments.push(VideoSegmentInfo { + path: segment_path, + index, + duration, + file_size: Some(file_size), + }); + } + } + + self.completed_segments.sort_by_key(|s| s.index); + } + + fn finalize_manifest(&self) { + let total_duration: Duration = self.completed_segments.iter().map(|s| s.duration).sum(); + + let manifest = Manifest { + version: MANIFEST_VERSION, + manifest_type: "m4s_segments", + init_segment: Some(INIT_SEGMENT_NAME.to_string()), + segments: self + .completed_segments + .iter() + .map(|s| SegmentEntry { + path: s + .path + .file_name() + .unwrap_or_default() + .to_string_lossy() + .into_owned(), + index: s.index, + duration: s.duration.as_secs_f64(), + is_complete: true, + file_size: s.file_size, + }) + .collect(), + total_duration: Some(total_duration.as_secs_f64()), + is_complete: true, + }; + + let manifest_path = self.base_path.join("manifest.json"); + if let Err(e) = atomic_write_json(&manifest_path, &manifest) { + tracing::warn!( + "Failed to write final manifest to {}: {e}", + manifest_path.display() + ); + } + } + + pub fn completed_segments(&self) -> &[VideoSegmentInfo] { + &self.completed_segments + } + + pub fn current_encoder(&self) -> Option<&H264Encoder> { + Some(&self.encoder) + } + + pub fn current_encoder_mut(&mut self) -> Option<&mut H264Encoder> { + Some(&mut self.encoder) + } + + pub fn base_path(&self) -> &Path { + &self.base_path + } + + pub fn segment_duration(&self) -> Duration { + self.segment_duration + } + + pub fn current_index(&self) -> u32 { + self.current_index + } + + pub fn init_segment_path(&self) -> PathBuf { + self.base_path.join(INIT_SEGMENT_NAME) + } +} diff --git a/crates/enc-ffmpeg/src/remux.rs b/crates/enc-ffmpeg/src/remux.rs index cf858002e2..027c7cefd0 100644 --- a/crates/enc-ffmpeg/src/remux.rs +++ b/crates/enc-ffmpeg/src/remux.rs @@ -103,16 +103,10 @@ fn open_input_with_format( } } -fn concatenate_with_concat_demuxer( - concat_list_path: &Path, - output: &Path, +fn remux_streams( + ictx: &mut avformat::context::Input, + octx: &mut avformat::context::Output, ) -> Result<(), RemuxError> { - let mut options = ffmpeg::Dictionary::new(); - options.set("safe", "0"); - - let mut ictx = open_input_with_format(concat_list_path, "concat", options)?; - let mut octx = avformat::output(output)?; - let mut stream_mapping: Vec> = Vec::new(); let mut output_stream_index = 0usize; @@ -171,7 +165,7 @@ fn concatenate_with_concat_demuxer( packet.set_stream(output_index); packet.set_position(-1); - packet.write_interleaved(&mut octx)?; + packet.write_interleaved(octx)?; } } @@ -180,6 +174,19 @@ fn concatenate_with_concat_demuxer( Ok(()) } +fn concatenate_with_concat_demuxer( + concat_list_path: &Path, + output: &Path, +) -> Result<(), RemuxError> { + let mut options = ffmpeg::Dictionary::new(); + options.set("safe", "0"); + + let mut ictx = open_input_with_format(concat_list_path, "concat", options)?; + let mut octx = avformat::output(output)?; + + remux_streams(&mut ictx, &mut octx) +} + pub fn concatenate_audio_to_ogg(fragments: &[PathBuf], output: &Path) -> Result<(), RemuxError> { if fragments.is_empty() { return Err(RemuxError::NoFragments); @@ -364,3 +371,90 @@ pub fn get_video_fps(path: &Path) -> Option { } Some((rate.numerator() as f64 / rate.denominator() as f64).round() as u32) } + +pub fn probe_m4s_can_decode_with_init( + init_path: &Path, + segment_path: &Path, +) -> Result { + let temp_path = segment_path.with_extension("probe_temp.mp4"); + + let init_data = std::fs::read(init_path) + .map_err(|e| format!("Failed to read init segment {}: {e}", init_path.display()))?; + let segment_data = std::fs::read(segment_path) + .map_err(|e| format!("Failed to read segment {}: {e}", segment_path.display()))?; + + { + let mut temp_file = std::fs::File::create(&temp_path) + .map_err(|e| format!("Failed to create temp file: {e}"))?; + temp_file + .write_all(&init_data) + .map_err(|e| format!("Failed to write init data: {e}"))?; + temp_file + .write_all(&segment_data) + .map_err(|e| format!("Failed to write segment data: {e}"))?; + temp_file + .sync_all() + .map_err(|e| format!("Failed to sync temp file: {e}"))?; + } + + let result = probe_video_can_decode(&temp_path); + + if let Err(e) = std::fs::remove_file(&temp_path) { + tracing::warn!("failed to remove temp file {}: {}", temp_path.display(), e); + } + + result +} + +pub fn concatenate_m4s_segments_with_init( + init_path: &Path, + segments: &[PathBuf], + output: &Path, +) -> Result<(), RemuxError> { + if segments.is_empty() { + return Err(RemuxError::NoFragments); + } + + if !init_path.exists() { + return Err(RemuxError::FragmentNotFound(init_path.to_path_buf())); + } + + for segment in segments { + if !segment.exists() { + return Err(RemuxError::FragmentNotFound(segment.clone())); + } + } + + let combined_path = output.with_extension("combined_fmp4.mp4"); + + { + let init_data = std::fs::read(init_path)?; + let mut combined_file = std::fs::File::create(&combined_path)?; + combined_file.write_all(&init_data)?; + + for segment in segments { + let segment_data = std::fs::read(segment)?; + combined_file.write_all(&segment_data)?; + } + combined_file.sync_all()?; + } + + let result = remux_to_regular_mp4(&combined_path, output); + + if let Err(e) = std::fs::remove_file(&combined_path) { + tracing::warn!( + "failed to remove combined file {}: {}", + combined_path.display(), + e + ); + } + + result +} + +fn remux_to_regular_mp4(input_path: &Path, output_path: &Path) -> Result<(), RemuxError> { + let mut ictx = avformat::input(input_path)?; + let mut octx = avformat::output(output_path)?; + + remux_streams(&mut ictx, &mut octx) +} diff --git a/crates/enc-ffmpeg/src/video/h264.rs b/crates/enc-ffmpeg/src/video/h264.rs index fa440c6f93..bd566f2e34 100644 --- a/crates/enc-ffmpeg/src/video/h264.rs +++ b/crates/enc-ffmpeg/src/video/h264.rs @@ -8,7 +8,7 @@ use ffmpeg::{ frame, threading::Config, }; -use tracing::{debug, error, trace}; +use tracing::{debug, error, trace, warn}; use crate::base::EncoderBase; @@ -120,7 +120,23 @@ impl H264EncoderBuilder { self.external_conversion, ) { Ok(encoder) => { - debug!("Using encoder {}", codec_name); + let is_hardware = matches!( + codec_name.as_str(), + "h264_videotoolbox" | "h264_nvenc" | "h264_qsv" | "h264_amf" | "h264_mf" + ); + if is_hardware { + debug!( + encoder = %codec_name, + "Selected hardware H264 encoder" + ); + } else { + warn!( + encoder = %codec_name, + input_width = input_config.width, + input_height = input_config.height, + "WARNING: Using SOFTWARE H264 encoder (high CPU usage expected)" + ); + } return Ok(encoder); } Err(err) => { @@ -156,16 +172,18 @@ impl H264EncoderBuilder { input_config.pixel_format } else { needs_pixel_conversion = true; - let format = ffmpeg::format::Pixel::NV12; - if !external_conversion { - debug!( - "Converting from {:?} to {:?} for H264 encoding", - input_config.pixel_format, format - ); - } - format + ffmpeg::format::Pixel::NV12 }; + debug!( + encoder = %codec.name(), + input_format = ?input_config.pixel_format, + output_format = ?output_format, + needs_pixel_conversion = needs_pixel_conversion, + external_conversion = external_conversion, + "Encoder pixel format configuration" + ); + if is_420(output_format) && (!output_width.is_multiple_of(2) || !output_height.is_multiple_of(2)) { @@ -187,8 +205,10 @@ impl H264EncoderBuilder { let converter = if external_conversion { debug!( - "External conversion enabled, skipping internal converter. Expected input: {:?} {}x{}", - output_format, output_width, output_height + output_format = ?output_format, + output_width = output_width, + output_height = output_height, + "External conversion enabled, skipping internal converter" ); None } else if needs_pixel_conversion || needs_scaling { @@ -207,7 +227,18 @@ impl H264EncoderBuilder { output_height, flags, ) { - Ok(context) => Some(context), + Ok(context) => { + debug!( + encoder = %codec.name(), + src_format = ?input_config.pixel_format, + src_size = %format!("{}x{}", input_config.width, input_config.height), + dst_format = ?output_format, + dst_size = %format!("{}x{}", output_width, output_height), + needs_scaling = needs_scaling, + "Created SOFTWARE scaler for pixel format conversion (CPU-intensive)" + ); + Some(context) + } Err(e) => { if needs_pixel_conversion { error!( @@ -223,6 +254,10 @@ impl H264EncoderBuilder { } } } else { + debug!( + encoder = %codec.name(), + "No pixel format conversion needed (zero-copy path)" + ); None }; @@ -351,6 +386,36 @@ impl H264Encoder { Ok(()) } + pub fn queue_frame_reusable( + &mut self, + frame: &mut frame::Video, + converted_frame: &mut Option, + timestamp: Duration, + output: &mut format::context::Output, + ) -> Result<(), QueueFrameError> { + self.base.update_pts(frame, timestamp, &mut self.encoder); + + let frame_to_send = if let Some(converter) = &mut self.converter { + let pts = frame.pts(); + let converted = converted_frame.get_or_insert_with(|| { + frame::Video::new(self.output_format, self.output_width, self.output_height) + }); + converter + .run(frame, converted) + .map_err(QueueFrameError::Converter)?; + converted.set_pts(pts); + converted as &frame::Video + } else { + frame as &frame::Video + }; + + self.base + .send_frame(frame_to_send, output, &mut self.encoder) + .map_err(QueueFrameError::Encode)?; + + Ok(()) + } + pub fn queue_preconverted_frame( &mut self, mut frame: frame::Video, diff --git a/crates/export/src/mp4.rs b/crates/export/src/mp4.rs index 080c53cb48..a6e22c2311 100644 --- a/crates/export/src/mp4.rs +++ b/crates/export/src/mp4.rs @@ -95,22 +95,15 @@ impl Mp4ExportSettings { info!("Created MP4File encoder"); - let mut encoded_frames = 0; while let Ok(frame) = frame_rx.recv() { encoder - .queue_video_frame( - frame.video, - Duration::from_secs_f32(encoded_frames as f32 / fps as f32), - ) + .queue_video_frame(frame.video, Duration::MAX) .map_err(|err| err.to_string())?; - encoded_frames += 1; if let Some(audio) = frame.audio { encoder.queue_audio_frame(audio); } } - info!("Encoded {encoded_frames} video frames"); - let res = encoder .finish() .map_err(|e| format!("Failed to finish encoding: {e}"))?; @@ -132,9 +125,9 @@ impl Mp4ExportSettings { async move { let mut frame_count = 0; let mut first_frame = None; - - let audio_samples_per_frame = - (f64::from(AudioRenderer::SAMPLE_RATE) / f64::from(fps)).ceil() as usize; + let sample_rate = u64::from(AudioRenderer::SAMPLE_RATE); + let fps_u64 = u64::from(fps); + let mut audio_sample_cursor = 0u64; loop { let (frame, frame_number) = @@ -160,14 +153,20 @@ impl Mp4ExportSettings { } } - let audio_frame = audio_renderer - .as_mut() - .and_then(|audio| audio.render_frame(audio_samples_per_frame, &project)) - .map(|mut frame| { - let pts = ((frame_number * frame.rate()) as f64 / fps as f64) as i64; + let audio_frame = audio_renderer.as_mut().and_then(|audio| { + let n = u64::from(frame_number); + let end = ((n + 1) * sample_rate) / fps_u64; + if end <= audio_sample_cursor { + return None; + } + let pts = audio_sample_cursor as i64; + let samples = (end - audio_sample_cursor) as usize; + audio_sample_cursor = end; + audio.render_frame(samples, &project).map(|mut frame| { frame.set_pts(Some(pts)); frame - }); + }) + }); if frame_tx .send(MP4Input { @@ -258,3 +257,29 @@ impl Mp4ExportSettings { Ok(output_path) } } + +#[cfg(test)] +mod tests { + use super::*; + + fn sum_samples(sample_rate: u64, fps: u64, frames: u64) -> u64 { + (0..frames) + .map(|n| { + let start = (n * sample_rate) / fps; + let end = ((n + 1) * sample_rate) / fps; + end - start + }) + .sum() + } + + #[test] + fn audio_samples_match_duration_across_fps() { + let sample_rate = u64::from(AudioRenderer::SAMPLE_RATE); + + for fps in [24u64, 30, 60, 90, 120, 144] { + let frames = fps * 10; + let expected = (frames * sample_rate) / fps; + assert_eq!(sum_samples(sample_rate, fps, frames), expected); + } + } +} diff --git a/crates/recording/Cargo.toml b/crates/recording/Cargo.toml index 36aceb0a2e..409c19cb0a 100644 --- a/crates/recording/Cargo.toml +++ b/crates/recording/Cargo.toml @@ -62,6 +62,7 @@ objc2-app-kit = "0.3.1" core-graphics = "0.24.0" core-foundation = "0.10.0" foreign-types-shared = "0.3" +libc = "0.2" scap-screencapturekit = { path = "../scap-screencapturekit" } cap-enc-avfoundation = { path = "../enc-avfoundation" } @@ -87,3 +88,6 @@ scap-cpal = { path = "../scap-cpal" } [dev-dependencies] tempfile = "3.20.0" tracing-subscriber = { version = "0.3.19", features = ["env-filter"] } + +[target.'cfg(target_os = "macos")'.dev-dependencies] +libproc = "0.14" diff --git a/crates/recording/examples/memory-leak-detector.rs b/crates/recording/examples/memory-leak-detector.rs index 0e77765732..efb5c70045 100644 --- a/crates/recording/examples/memory-leak-detector.rs +++ b/crates/recording/examples/memory-leak-detector.rs @@ -18,106 +18,19 @@ const DEFAULT_DURATION_SECS: u64 = 120; #[cfg(target_os = "macos")] fn get_memory_usage() -> Option { - use std::process::Command; + use libproc::libproc::pid_rusage::{RUsageInfoV4, pidrusage}; - let pid = std::process::id(); - - let ps_output = Command::new("ps") - .args(["-o", "rss=,vsz=", "-p", &pid.to_string()]) - .output() - .ok()?; - - let stdout = String::from_utf8_lossy(&ps_output.stdout); - let parts: Vec<&str> = stdout.split_whitespace().collect(); - - let (rss_mb, vsz_mb) = if parts.len() >= 2 { - let rss_kb: u64 = parts[0].parse().ok()?; - let vsz_kb: u64 = parts[1].parse().ok()?; - (rss_kb as f64 / 1024.0, vsz_kb as f64 / 1024.0) - } else { - return None; - }; - - let (footprint_mb, dirty_mb) = Command::new("footprint") - .arg(pid.to_string()) - .output() - .ok() - .filter(|o| o.status.success()) - .and_then(|output| { - let stdout = String::from_utf8_lossy(&output.stdout); - parse_footprint_values(&stdout) - }) - .unwrap_or((None, None)); + let pid = std::process::id() as i32; + let rusage: RUsageInfoV4 = pidrusage(pid).ok()?; Some(MemoryStats { - resident_mb: rss_mb, - virtual_mb: vsz_mb, - footprint_mb, - dirty_mb, + resident_mb: rusage.ri_resident_size as f64 / 1024.0 / 1024.0, + footprint_mb: Some(rusage.ri_phys_footprint as f64 / 1024.0 / 1024.0), + dirty_mb: None, compressed_mb: None, }) } -#[cfg(target_os = "macos")] -fn parse_footprint_values(output: &str) -> Option<(Option, Option)> { - let mut footprint_kb: Option = None; - let mut dirty_kb: Option = None; - - for line in output.lines() { - if line.contains("phys_footprint:") { - let parts: Vec<&str> = line.split_whitespace().collect(); - if parts.len() >= 2 { - footprint_kb = parse_size_kb(parts[1]); - } - } else if line.contains("TOTAL") && dirty_kb.is_none() { - let parts: Vec<&str> = line.split_whitespace().collect(); - if !parts.is_empty() { - dirty_kb = parse_size_kb(parts[0]); - } - } - } - - Some(( - footprint_kb.map(|v| v / 1024.0), - dirty_kb.map(|v| v / 1024.0), - )) -} - -#[cfg(target_os = "macos")] -fn parse_size_kb(s: &str) -> Option { - let s = s.trim(); - if s.ends_with("KB") || s.ends_with("kb") { - s.trim_end_matches("KB") - .trim_end_matches("kb") - .trim() - .parse() - .ok() - } else if s.ends_with("MB") || s.ends_with("mb") { - s.trim_end_matches("MB") - .trim_end_matches("mb") - .trim() - .parse::() - .ok() - .map(|v| v * 1024.0) - } else if s.ends_with("GB") || s.ends_with("gb") { - s.trim_end_matches("GB") - .trim_end_matches("gb") - .trim() - .parse::() - .ok() - .map(|v| v * 1024.0 * 1024.0) - } else if s.ends_with('B') || s.ends_with('b') { - s.trim_end_matches('B') - .trim_end_matches('b') - .trim() - .parse::() - .ok() - .map(|v| v / 1024.0) - } else { - s.parse().ok() - } -} - #[cfg(not(target_os = "macos"))] fn get_memory_usage() -> Option { None @@ -126,7 +39,6 @@ fn get_memory_usage() -> Option { #[derive(Debug, Clone, Copy)] struct MemoryStats { resident_mb: f64, - virtual_mb: f64, footprint_mb: Option, #[allow(dead_code)] dirty_mb: Option, @@ -136,11 +48,11 @@ struct MemoryStats { impl MemoryStats { fn primary_metric(&self) -> f64 { - self.resident_mb + self.footprint_mb.unwrap_or(self.resident_mb) } fn metric_name() -> &'static str { - "RSS" + "Footprint" } } @@ -170,10 +82,10 @@ impl MemoryTracker { if let Some(baseline) = self.baseline { println!( - "Baseline: {:.1} MB {} (Footprint: {:.1} MB)", + "Baseline: {:.1} MB {} (RSS: {:.1} MB)", baseline.primary_metric(), MemoryStats::metric_name(), - baseline.footprint_mb.unwrap_or(0.0) + baseline.resident_mb ); } @@ -195,10 +107,10 @@ impl MemoryTracker { println!("\nMemory Timeline:"); println!( - "{:>8} {:>12} {:>12} {:>12} {:>12}", - "Time(s)", "RSS(MB)", "Delta", "Footprint", "VSZ(MB)" + "{:>8} {:>14} {:>10} {:>12}", + "Time(s)", "Footprint(MB)", "Delta", "RSS(MB)" ); - println!("{:-<70}", ""); + println!("{:-<50}", ""); let mut prev_memory = first.1.primary_metric(); for (time, stats) in &self.samples { @@ -210,20 +122,19 @@ impl MemoryTracker { "~0".to_string() }; println!( - "{:>8.1} {:>12.1} {:>12} {:>12.1} {:>12.1}", + "{:>8.1} {:>14.1} {:>10} {:>12.1}", time.as_secs_f64(), current, delta_str, - stats.footprint_mb.unwrap_or(0.0), - stats.virtual_mb + stats.resident_mb ); prev_memory = current; } println!("\n=== Summary ==="); println!("Duration: {duration_secs:.1}s"); - println!("Start RSS: {:.1} MB", first.1.primary_metric()); - println!("End RSS: {:.1} MB", last.1.primary_metric()); + println!("Start Footprint: {:.1} MB", first.1.primary_metric()); + println!("End Footprint: {:.1} MB", last.1.primary_metric()); println!("Total growth: {memory_growth:.1} MB"); println!( "Growth rate: {:.2} MB/s ({:.1} MB/10s)", @@ -251,7 +162,7 @@ impl MemoryTracker { } println!( - "\nPeak RSS: {:.1} MB", + "\nPeak Footprint: {:.1} MB", self.samples .iter() .map(|(_, s)| s.primary_metric()) @@ -344,7 +255,7 @@ async fn run_memory_test( ) .await?; - let sample_interval = Duration::from_secs(5); + let sample_interval = Duration::from_secs(1); let mut next_sample = start + sample_interval; while start.elapsed() < Duration::from_secs(duration_secs) { @@ -355,11 +266,10 @@ async fn run_memory_test( let current = get_memory_usage(); if let Some(stats) = current { println!( - "[{:>5.1}s] RSS: {:.1} MB, Footprint: {:.1} MB, VSZ: {:.1} MB", + "[{:>5.1}s] Footprint: {:.1} MB, RSS: {:.1} MB", start.elapsed().as_secs_f64(), - stats.resident_mb, stats.footprint_mb.unwrap_or(0.0), - stats.virtual_mb + stats.resident_mb ); } next_sample = Instant::now() + sample_interval; @@ -407,7 +317,7 @@ async fn run_camera_only_test(duration_secs: u64) -> Result<(), Box Result<(), Box5.1}s] RSS: {:.1} MB, Footprint: {:.1} MB, Frames: {}, Queue: {}", + "[{:>5.1}s] Footprint: {:.1} MB, RSS: {:.1} MB, Frames: {}, Queue: {}", start.elapsed().as_secs_f64(), - stats.resident_mb, stats.footprint_mb.unwrap_or(0.0), + stats.resident_mb, frame_count, queue_len ); diff --git a/crates/recording/src/capture_pipeline.rs b/crates/recording/src/capture_pipeline.rs index 1caa3afe8e..7f745d3470 100644 --- a/crates/recording/src/capture_pipeline.rs +++ b/crates/recording/src/capture_pipeline.rs @@ -1,4 +1,5 @@ use crate::{ + SharedPauseState, feeds::microphone::MicrophoneFeedLock, output_pipeline::*, sources, @@ -6,7 +7,7 @@ use crate::{ }; #[cfg(target_os = "macos")] -use crate::output_pipeline::{MacOSSegmentedMuxer, MacOSSegmentedMuxerConfig}; +use crate::output_pipeline::{MacOSFragmentedM4SMuxer, MacOSFragmentedM4SMuxerConfig}; use anyhow::anyhow; use cap_timestamp::Timestamps; use std::{path::PathBuf, sync::Arc}; @@ -50,6 +51,7 @@ pub trait MakeCapturePipeline: ScreenCaptureFormat + std::fmt::Debug + 'static { output_path: PathBuf, start_time: Timestamps, fragmented: bool, + shared_pause_state: Option, #[cfg(windows)] encoder_preferences: EncoderPreferences, ) -> anyhow::Result where @@ -76,6 +78,7 @@ impl MakeCapturePipeline for screen_capture::CMSampleBufferCapture { output_path: PathBuf, start_time: Timestamps, fragmented: bool, + shared_pause_state: Option, ) -> anyhow::Result { if fragmented { let fragments_dir = output_path @@ -86,7 +89,10 @@ impl MakeCapturePipeline for screen_capture::CMSampleBufferCapture { OutputPipeline::builder(fragments_dir) .with_video::(screen_capture) .with_timestamps(start_time) - .build::(MacOSSegmentedMuxerConfig::default()) + .build::(MacOSFragmentedM4SMuxerConfig { + shared_pause_state, + ..Default::default() + }) .await } else { OutputPipeline::builder(output_path.clone()) @@ -130,6 +136,7 @@ impl MakeCapturePipeline for screen_capture::Direct3DCapture { output_path: PathBuf, start_time: Timestamps, fragmented: bool, + _shared_pause_state: Option, encoder_preferences: EncoderPreferences, ) -> anyhow::Result { let d3d_device = screen_capture.d3d_device.clone(); diff --git a/crates/recording/src/cursor.rs b/crates/recording/src/cursor.rs index a3701dfeaa..796093d035 100644 --- a/crates/recording/src/cursor.rs +++ b/crates/recording/src/cursor.rs @@ -72,11 +72,8 @@ pub fn spawn_cursor_recorder( use cap_utils::spawn_actor; use device_query::{DeviceQuery, DeviceState}; use futures::future::Either; - use std::{ - hash::{DefaultHasher, Hash, Hasher}, - pin::pin, - time::Duration, - }; + use sha2::{Digest, Sha256}; + use std::{pin::pin, time::Duration}; use tracing::{error, info}; let stop_token = CancellationToken::new(); @@ -100,9 +97,10 @@ pub fn spawn_cursor_recorder( let mut last_flush = Instant::now(); let flush_interval = Duration::from_secs(CURSOR_FLUSH_INTERVAL_SECS); + let mut last_cursor_id: Option = None; loop { - let sleep = tokio::time::sleep(Duration::from_millis(10)); + let sleep = tokio::time::sleep(Duration::from_millis(16)); let Either::Right(_) = futures::future::select(pin!(stop_token_child.cancelled()), pin!(sleep)).await else { @@ -112,13 +110,22 @@ pub fn spawn_cursor_recorder( let elapsed = start_time.instant().elapsed().as_secs_f64() * 1000.0; let mouse_state = device_state.get_mouse(); - let cursor_data = get_cursor_data(); - let cursor_id = if let Some(data) = cursor_data { - let mut hasher = DefaultHasher::default(); - data.image.hash(&mut hasher); - let id = hasher.finish(); + let position = cap_cursor_capture::RawCursorPosition::get(); + let position_changed = position != last_position; + + if position_changed { + last_position = position; + } - if let Some(existing_id) = response.cursors.get(&id) { + let cursor_id = if let Some(data) = get_cursor_data() { + let hash_bytes = Sha256::digest(&data.image); + let id = u64::from_le_bytes( + hash_bytes[..8] + .try_into() + .expect("sha256 produces at least 8 bytes"), + ); + + let cursor_id = if let Some(existing_id) = response.cursors.get(&id) { existing_id.id.to_string() } else { let cursor_id = response.next_cursor_id.to_string(); @@ -146,34 +153,33 @@ pub fn spawn_cursor_recorder( } cursor_id - } + }; + last_cursor_id = Some(cursor_id.clone()); + Some(cursor_id) } else { - "default".to_string() + last_cursor_id.clone() }; - let position = cap_cursor_capture::RawCursorPosition::get(); - - let position = (position != last_position).then(|| { - last_position = position; + let Some(cursor_id) = cursor_id else { + continue; + }; + if position_changed { let cropped_norm_pos = position - .relative_to_display(display)? - .normalize()? - .with_crop(crop_bounds); - - Some((cropped_norm_pos.x(), cropped_norm_pos.y())) - }); - - if let Some((x, y)) = position.flatten() { - let mouse_event = CursorMoveEvent { - active_modifiers: vec![], - cursor_id: cursor_id.clone(), - time_ms: elapsed, - x, - y, - }; - - response.moves.push(mouse_event); + .relative_to_display(display) + .and_then(|p| p.normalize()) + .map(|p| p.with_crop(crop_bounds)); + + if let Some(pos) = cropped_norm_pos { + let mouse_event = CursorMoveEvent { + active_modifiers: vec![], + cursor_id: cursor_id.clone(), + time_ms: elapsed, + x: pos.x(), + y: pos.y(), + }; + response.moves.push(mouse_event); + } } for (num, &pressed) in mouse_state.button_pressed.iter().enumerate() { diff --git a/crates/recording/src/output_pipeline/core.rs b/crates/recording/src/output_pipeline/core.rs index 44148ec89c..d4f5f6204f 100644 --- a/crates/recording/src/output_pipeline/core.rs +++ b/crates/recording/src/output_pipeline/core.rs @@ -17,7 +17,7 @@ use std::{ path::{Path, PathBuf}, sync::{ Arc, - atomic::{self, AtomicBool}, + atomic::{self, AtomicBool, Ordering}, }, time::Duration, }; @@ -25,6 +25,67 @@ use tokio::task::JoinHandle; use tokio_util::sync::{CancellationToken, DropGuard}; use tracing::*; +struct SharedPauseStateInner { + paused_at: Option, + offset: Duration, +} + +#[derive(Clone)] +pub struct SharedPauseState { + flag: Arc, + inner: Arc>, +} + +impl SharedPauseState { + pub fn new(flag: Arc) -> Self { + Self { + flag, + inner: Arc::new(std::sync::Mutex::new(SharedPauseStateInner { + paused_at: None, + offset: Duration::ZERO, + })), + } + } + + pub fn adjust(&self, timestamp: Duration) -> anyhow::Result> { + let mut inner = self + .inner + .lock() + .map_err(|e| anyhow!("Lock poisoned: {e}"))?; + + if self.flag.load(Ordering::Relaxed) { + if inner.paused_at.is_none() { + inner.paused_at = Some(timestamp); + } + return Ok(None); + } + + if let Some(start) = inner.paused_at.take() { + let delta = timestamp.checked_sub(start).ok_or_else(|| { + anyhow!( + "Frame timestamp went backward during unpause (resume={start:?}, current={timestamp:?})" + ) + })?; + + inner.offset = inner.offset.checked_add(delta).ok_or_else(|| { + anyhow!( + "Pause offset overflow (offset={:?}, delta={delta:?})", + inner.offset + ) + })?; + } + + let adjusted = timestamp.checked_sub(inner.offset).ok_or_else(|| { + anyhow!( + "Adjusted timestamp underflow (timestamp={timestamp:?}, offset={:?})", + inner.offset + ) + })?; + + Ok(Some(adjusted)) + } +} + pub struct OnceSender(Option>); impl OnceSender { @@ -455,7 +516,6 @@ fn spawn_video_encoder, TVideo: V let mut first_tx = Some(first_tx); let mut frame_count = 0u64; - let res = stop_token .run_until_cancelled(async { while let Some(frame) = video_rx.next().await { @@ -487,9 +547,21 @@ fn spawn_video_encoder, TVideo: V if was_cancelled { info!("mux-video cancelled, draining remaining frames from channel"); + let drain_start = std::time::Instant::now(); + let drain_timeout = Duration::from_secs(2); + let max_drain_frames = 30u64; let mut drained = 0u64; + let mut skipped = 0u64; + + let mut hit_limit = false; while let Some(frame) = video_rx.next().await { frame_count += 1; + + if drain_start.elapsed() > drain_timeout || drained >= max_drain_frames { + hit_limit = true; + break; + } + drained += 1; let timestamp = frame.timestamp(); @@ -506,14 +578,18 @@ fn spawn_video_encoder, TVideo: V Ok(()) => {} Err(e) => { warn!("Error processing drained frame: {e}"); - break; + skipped += 1; } } } - if drained > 0 { + + if drained > 0 || skipped > 0 || hit_limit { info!( - "mux-video drained {} additional frames after cancellation", - drained + "mux-video drain complete: {} frames processed, {} errors (limit hit: {}) in {:?}", + drained, + skipped, + hit_limit, + drain_start.elapsed() ); } } diff --git a/crates/recording/src/output_pipeline/ffmpeg.rs b/crates/recording/src/output_pipeline/ffmpeg.rs index 1c8e2fe694..8e94393317 100644 --- a/crates/recording/src/output_pipeline/ffmpeg.rs +++ b/crates/recording/src/output_pipeline/ffmpeg.rs @@ -1,5 +1,5 @@ use crate::{ - TaskPool, + SharedPauseState, TaskPool, output_pipeline::{AudioFrame, AudioMuxer, Muxer, VideoFrame, VideoMuxer}, }; use anyhow::{Context, anyhow}; @@ -197,16 +197,21 @@ impl AudioMuxer for FragmentedAudioMuxer { } } -pub struct SegmentedAudioMuxer(SegmentedAudioEncoder); +pub struct SegmentedAudioMuxer { + encoder: SegmentedAudioEncoder, + pause: Option, +} pub struct SegmentedAudioMuxerConfig { pub segment_duration: Duration, + pub shared_pause_state: Option, } impl Default for SegmentedAudioMuxerConfig { fn default() -> Self { Self { segment_duration: Duration::from_secs(3), + shared_pause_state: None, } } } @@ -228,14 +233,19 @@ impl Muxer for SegmentedAudioMuxer { let audio_config = audio_config.ok_or_else(|| anyhow!("No audio configuration provided"))?; - Ok(Self( - SegmentedAudioEncoder::init(output_path, audio_config, config.segment_duration) - .map_err(|e| anyhow!("Failed to initialize segmented audio encoder: {e}"))?, - )) + Ok(Self { + encoder: SegmentedAudioEncoder::init( + output_path, + audio_config, + config.segment_duration, + ) + .map_err(|e| anyhow!("Failed to initialize segmented audio encoder: {e}"))?, + pause: config.shared_pause_state, + }) } fn finish(&mut self, timestamp: Duration) -> anyhow::Result> { - self.0 + self.encoder .finish_with_timestamp(timestamp) .map_err(Into::into) .map(|_| Ok(())) @@ -244,8 +254,17 @@ impl Muxer for SegmentedAudioMuxer { impl AudioMuxer for SegmentedAudioMuxer { fn send_audio_frame(&mut self, frame: AudioFrame, timestamp: Duration) -> anyhow::Result<()> { - self.0 - .queue_frame(frame.inner, timestamp) + let adjusted_timestamp = if let Some(pause) = &self.pause { + match pause.adjust(timestamp)? { + Some(ts) => ts, + None => return Ok(()), + } + } else { + timestamp + }; + + self.encoder + .queue_frame(frame.inner, adjusted_timestamp) .map_err(|e| anyhow!("Failed to queue audio frame: {e}")) } } diff --git a/crates/recording/src/output_pipeline/fragmented.rs b/crates/recording/src/output_pipeline/fragmented.rs deleted file mode 100644 index a5dd3d933a..0000000000 --- a/crates/recording/src/output_pipeline/fragmented.rs +++ /dev/null @@ -1,294 +0,0 @@ -#[cfg(target_os = "macos")] -use crate::{ - VideoFrame, - output_pipeline::{AudioFrame, AudioMuxer, Muxer, TaskPool, VideoMuxer}, - sources::screen_capture, -}; - -#[cfg(target_os = "macos")] -use anyhow::anyhow; -#[cfg(target_os = "macos")] -use cap_enc_avfoundation::SegmentedMP4Encoder; -#[cfg(target_os = "macos")] -use cap_media_info::{AudioInfo, VideoInfo}; -#[cfg(target_os = "macos")] -use cap_timestamp::Timestamp; -#[cfg(target_os = "macos")] -use std::{ - path::PathBuf, - sync::{Arc, atomic::AtomicBool}, - time::Duration, -}; - -#[cfg(target_os = "macos")] -pub struct FragmentedAVFoundationMp4Muxer { - inner: SegmentedMP4Encoder, - pause_flag: Arc, -} - -#[cfg(target_os = "macos")] -pub struct FragmentedAVFoundationMp4MuxerConfig { - pub output_height: Option, - pub segment_duration: Duration, -} - -#[cfg(target_os = "macos")] -impl Default for FragmentedAVFoundationMp4MuxerConfig { - fn default() -> Self { - Self { - output_height: None, - segment_duration: Duration::from_secs(3), - } - } -} - -#[cfg(target_os = "macos")] -impl FragmentedAVFoundationMp4Muxer { - const MAX_QUEUE_RETRIES: u32 = 1500; -} - -#[cfg(target_os = "macos")] -#[derive(Clone)] -pub struct FragmentedNativeCameraFrame { - pub sample_buf: cidre::arc::R, - pub timestamp: Timestamp, -} - -#[cfg(target_os = "macos")] -unsafe impl Send for FragmentedNativeCameraFrame {} -#[cfg(target_os = "macos")] -unsafe impl Sync for FragmentedNativeCameraFrame {} - -#[cfg(target_os = "macos")] -impl VideoFrame for FragmentedNativeCameraFrame { - fn timestamp(&self) -> Timestamp { - self.timestamp - } -} - -#[cfg(target_os = "macos")] -impl Muxer for FragmentedAVFoundationMp4Muxer { - type Config = FragmentedAVFoundationMp4MuxerConfig; - - async fn setup( - config: Self::Config, - output_path: PathBuf, - video_config: Option, - audio_config: Option, - pause_flag: Arc, - _tasks: &mut TaskPool, - ) -> anyhow::Result { - let video_config = - video_config.ok_or_else(|| anyhow!("Invariant: No video source provided"))?; - - Ok(Self { - inner: SegmentedMP4Encoder::init( - output_path, - video_config, - audio_config, - config.output_height, - config.segment_duration, - ) - .map_err(|e| anyhow!("{e}"))?, - pause_flag, - }) - } - - fn finish(&mut self, timestamp: Duration) -> anyhow::Result> { - Ok(self.inner.finish(Some(timestamp)).map(Ok)?) - } -} - -#[cfg(target_os = "macos")] -impl VideoMuxer for FragmentedAVFoundationMp4Muxer { - type VideoFrame = screen_capture::VideoFrame; - - fn send_video_frame( - &mut self, - frame: Self::VideoFrame, - timestamp: Duration, - ) -> anyhow::Result<()> { - if self.pause_flag.load(std::sync::atomic::Ordering::Relaxed) { - self.inner.pause(); - } else { - self.inner.resume(); - } - - let mut retry_count = 0; - loop { - match self - .inner - .queue_video_frame(frame.sample_buf.clone(), timestamp) - { - Ok(()) => break, - Err(cap_enc_avfoundation::QueueFrameError::NotReadyForMore) => { - retry_count += 1; - if retry_count >= Self::MAX_QUEUE_RETRIES { - return Err(anyhow!( - "send_video_frame/timeout after {} retries", - Self::MAX_QUEUE_RETRIES - )); - } - std::thread::sleep(Duration::from_millis(2)); - continue; - } - Err(e) => return Err(anyhow!("send_video_frame/{e}")), - } - } - - Ok(()) - } -} - -#[cfg(target_os = "macos")] -impl AudioMuxer for FragmentedAVFoundationMp4Muxer { - fn send_audio_frame(&mut self, frame: AudioFrame, timestamp: Duration) -> anyhow::Result<()> { - let mut retry_count = 0; - loop { - match self.inner.queue_audio_frame(&frame.inner, timestamp) { - Ok(()) => break, - Err(cap_enc_avfoundation::QueueFrameError::NotReadyForMore) => { - retry_count += 1; - if retry_count >= Self::MAX_QUEUE_RETRIES { - return Err(anyhow!( - "send_audio_frame/retries_exceeded after {} retries", - Self::MAX_QUEUE_RETRIES - )); - } - std::thread::sleep(Duration::from_millis(2)); - continue; - } - Err(e) => return Err(anyhow!("send_audio_frame/{e}")), - } - } - - Ok(()) - } -} - -#[cfg(target_os = "macos")] -pub struct FragmentedAVFoundationCameraMuxer { - inner: SegmentedMP4Encoder, - pause_flag: Arc, -} - -#[cfg(target_os = "macos")] -pub struct FragmentedAVFoundationCameraMuxerConfig { - pub output_height: Option, - pub segment_duration: Duration, -} - -#[cfg(target_os = "macos")] -impl Default for FragmentedAVFoundationCameraMuxerConfig { - fn default() -> Self { - Self { - output_height: None, - segment_duration: Duration::from_secs(3), - } - } -} - -#[cfg(target_os = "macos")] -impl FragmentedAVFoundationCameraMuxer { - const MAX_QUEUE_RETRIES: u32 = 1500; -} - -#[cfg(target_os = "macos")] -impl Muxer for FragmentedAVFoundationCameraMuxer { - type Config = FragmentedAVFoundationCameraMuxerConfig; - - async fn setup( - config: Self::Config, - output_path: PathBuf, - video_config: Option, - audio_config: Option, - pause_flag: Arc, - _tasks: &mut TaskPool, - ) -> anyhow::Result { - let video_config = - video_config.ok_or_else(|| anyhow!("Invariant: No video source provided"))?; - - Ok(Self { - inner: SegmentedMP4Encoder::init( - output_path, - video_config, - audio_config, - config.output_height, - config.segment_duration, - ) - .map_err(|e| anyhow!("{e}"))?, - pause_flag, - }) - } - - fn finish(&mut self, timestamp: Duration) -> anyhow::Result> { - Ok(self.inner.finish(Some(timestamp)).map(Ok)?) - } -} - -#[cfg(target_os = "macos")] -impl VideoMuxer for FragmentedAVFoundationCameraMuxer { - type VideoFrame = crate::output_pipeline::NativeCameraFrame; - - fn send_video_frame( - &mut self, - frame: Self::VideoFrame, - timestamp: Duration, - ) -> anyhow::Result<()> { - if self.pause_flag.load(std::sync::atomic::Ordering::Relaxed) { - self.inner.pause(); - } else { - self.inner.resume(); - } - - let mut retry_count = 0; - loop { - match self - .inner - .queue_video_frame(frame.sample_buf.clone(), timestamp) - { - Ok(()) => break, - Err(cap_enc_avfoundation::QueueFrameError::NotReadyForMore) => { - retry_count += 1; - if retry_count >= Self::MAX_QUEUE_RETRIES { - return Err(anyhow!( - "send_video_frame/timeout after {} retries", - Self::MAX_QUEUE_RETRIES - )); - } - std::thread::sleep(Duration::from_millis(2)); - continue; - } - Err(e) => return Err(anyhow!("send_video_frame/{e}")), - } - } - - Ok(()) - } -} - -#[cfg(target_os = "macos")] -impl AudioMuxer for FragmentedAVFoundationCameraMuxer { - fn send_audio_frame(&mut self, frame: AudioFrame, timestamp: Duration) -> anyhow::Result<()> { - let mut retry_count = 0; - loop { - match self.inner.queue_audio_frame(&frame.inner, timestamp) { - Ok(()) => break, - Err(cap_enc_avfoundation::QueueFrameError::NotReadyForMore) => { - retry_count += 1; - if retry_count >= Self::MAX_QUEUE_RETRIES { - return Err(anyhow!( - "send_audio_frame/retries_exceeded after {} retries", - Self::MAX_QUEUE_RETRIES - )); - } - std::thread::sleep(Duration::from_millis(2)); - continue; - } - Err(e) => return Err(anyhow!("send_audio_frame/{e}")), - } - } - - Ok(()) - } -} diff --git a/crates/recording/src/output_pipeline/macos_fragmented_m4s.rs b/crates/recording/src/output_pipeline/macos_fragmented_m4s.rs new file mode 100644 index 0000000000..44f1633abf --- /dev/null +++ b/crates/recording/src/output_pipeline/macos_fragmented_m4s.rs @@ -0,0 +1,874 @@ +use crate::{ + AudioFrame, AudioMuxer, Muxer, SharedPauseState, TaskPool, VideoMuxer, + output_pipeline::NativeCameraFrame, screen_capture, +}; +use anyhow::{Context, anyhow}; +use cap_enc_ffmpeg::h264::{H264EncoderBuilder, H264Preset}; +use cap_enc_ffmpeg::segmented_stream::{SegmentedVideoEncoder, SegmentedVideoEncoderConfig}; +use cap_media_info::{AudioInfo, VideoInfo}; +use std::{ + path::PathBuf, + sync::{ + Arc, Mutex, + atomic::AtomicBool, + mpsc::{SyncSender, sync_channel}, + }, + thread::JoinHandle, + time::Duration, +}; +use tracing::*; + +fn get_muxer_buffer_size() -> usize { + std::env::var("CAP_MUXER_BUFFER_SIZE") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(3) +} + +struct FrameDropTracker { + drops_in_window: u32, + frames_in_window: u32, + total_drops: u64, + total_frames: u64, + last_check: std::time::Instant, +} + +impl FrameDropTracker { + fn new() -> Self { + Self { + drops_in_window: 0, + frames_in_window: 0, + total_drops: 0, + total_frames: 0, + last_check: std::time::Instant::now(), + } + } + + fn record_frame(&mut self) { + self.frames_in_window += 1; + self.total_frames += 1; + self.check_drop_rate(); + } + + fn record_drop(&mut self) { + self.drops_in_window += 1; + self.total_drops += 1; + self.check_drop_rate(); + } + + fn check_drop_rate(&mut self) { + if self.last_check.elapsed() >= Duration::from_secs(5) { + let total_in_window = self.frames_in_window + self.drops_in_window; + if total_in_window > 0 { + let drop_rate = 100.0 * self.drops_in_window as f64 / total_in_window as f64; + if drop_rate > 5.0 { + warn!( + frames = self.frames_in_window, + drops = self.drops_in_window, + drop_rate_pct = format!("{:.1}%", drop_rate), + total_frames = self.total_frames, + total_drops = self.total_drops, + "M4S muxer frame drop rate exceeds 5% threshold" + ); + } else if self.drops_in_window > 0 { + debug!( + frames = self.frames_in_window, + drops = self.drops_in_window, + drop_rate_pct = format!("{:.1}%", drop_rate), + "M4S muxer frame stats" + ); + } + } + self.drops_in_window = 0; + self.frames_in_window = 0; + self.last_check = std::time::Instant::now(); + } + } +} + +struct EncoderState { + video_tx: SyncSender, Duration)>>, + encoder: Arc>, + encoder_handle: Option>>, +} + +pub struct MacOSFragmentedM4SMuxer { + base_path: PathBuf, + video_config: VideoInfo, + segment_duration: Duration, + preset: H264Preset, + output_size: Option<(u32, u32)>, + state: Option, + pause: SharedPauseState, + frame_drops: FrameDropTracker, + started: bool, +} + +pub struct MacOSFragmentedM4SMuxerConfig { + pub segment_duration: Duration, + pub preset: H264Preset, + pub output_size: Option<(u32, u32)>, + pub shared_pause_state: Option, +} + +impl Default for MacOSFragmentedM4SMuxerConfig { + fn default() -> Self { + Self { + segment_duration: Duration::from_secs(3), + preset: H264Preset::Ultrafast, + output_size: None, + shared_pause_state: None, + } + } +} + +impl Muxer for MacOSFragmentedM4SMuxer { + type Config = MacOSFragmentedM4SMuxerConfig; + + async fn setup( + config: Self::Config, + output_path: PathBuf, + video_config: Option, + _audio_config: Option, + pause_flag: Arc, + _tasks: &mut TaskPool, + ) -> anyhow::Result + where + Self: Sized, + { + let video_config = + video_config.ok_or_else(|| anyhow!("invariant: video config expected"))?; + + std::fs::create_dir_all(&output_path) + .with_context(|| format!("Failed to create segments directory: {output_path:?}"))?; + + let pause = config + .shared_pause_state + .unwrap_or_else(|| SharedPauseState::new(pause_flag)); + + Ok(Self { + base_path: output_path, + video_config, + segment_duration: config.segment_duration, + preset: config.preset, + output_size: config.output_size, + state: None, + pause, + frame_drops: FrameDropTracker::new(), + started: false, + }) + } + + fn stop(&mut self) { + if let Some(state) = &self.state + && let Err(e) = state.video_tx.send(None) + { + trace!("M4S encoder channel already closed during stop: {e}"); + } + } + + fn finish(&mut self, timestamp: Duration) -> anyhow::Result> { + if let Some(mut state) = self.state.take() { + if let Err(e) = state.video_tx.send(None) { + trace!("M4S encoder channel already closed during finish: {e}"); + } + + if let Some(handle) = state.encoder_handle.take() { + let timeout = Duration::from_secs(5); + let start = std::time::Instant::now(); + loop { + if handle.is_finished() { + match handle.join() { + Err(panic_payload) => { + warn!( + "M4S encoder thread panicked during finish: {:?}", + panic_payload + ); + } + Ok(Err(e)) => { + warn!("M4S encoder thread returned error: {e}"); + } + Ok(Ok(())) => {} + } + break; + } + if start.elapsed() > timeout { + warn!( + "M4S encoder thread did not finish within {:?}, abandoning", + timeout + ); + break; + } + std::thread::sleep(Duration::from_millis(50)); + } + } + + if let Ok(mut encoder) = state.encoder.lock() + && let Err(e) = encoder.finish_with_timestamp(timestamp) + { + warn!("Failed to finish segmented encoder: {e}"); + } + } + + Ok(Ok(())) + } +} + +impl MacOSFragmentedM4SMuxer { + fn start_encoder(&mut self) -> anyhow::Result<()> { + let buffer_size = get_muxer_buffer_size(); + debug!( + buffer_size = buffer_size, + "M4S muxer encoder channel buffer size" + ); + + let (video_tx, video_rx) = + sync_channel::, Duration)>>(buffer_size); + let (ready_tx, ready_rx) = sync_channel::>(1); + + let encoder_config = SegmentedVideoEncoderConfig { + segment_duration: self.segment_duration, + preset: self.preset, + bpp: H264EncoderBuilder::QUALITY_BPP, + output_size: self.output_size, + }; + + let encoder = + SegmentedVideoEncoder::init(self.base_path.clone(), self.video_config, encoder_config)?; + let encoder = Arc::new(Mutex::new(encoder)); + let encoder_clone = encoder.clone(); + let video_config = self.video_config; + + let encoder_handle = std::thread::Builder::new() + .name("m4s-segment-encoder".to_string()) + .spawn(move || { + let pixel_format = match video_config.pixel_format { + cap_media_info::Pixel::NV12 => ffmpeg::format::Pixel::NV12, + cap_media_info::Pixel::BGRA => ffmpeg::format::Pixel::BGRA, + cap_media_info::Pixel::UYVY422 => ffmpeg::format::Pixel::UYVY422, + _ => ffmpeg::format::Pixel::NV12, + }; + + let mut frame_pool = + FramePool::new(pixel_format, video_config.width, video_config.height); + + if ready_tx.send(Ok(())).is_err() { + return Err(anyhow!("Failed to send ready signal - receiver dropped")); + } + + let mut slow_convert_count = 0u32; + let mut slow_encode_count = 0u32; + let mut total_frames = 0u64; + const SLOW_THRESHOLD_MS: u128 = 5; + + while let Ok(Some((sample_buf, timestamp))) = video_rx.recv() { + let convert_start = std::time::Instant::now(); + let frame = frame_pool.get_frame(); + let fill_result = fill_frame_from_sample_buf(&sample_buf, frame); + let convert_elapsed_ms = convert_start.elapsed().as_millis(); + + if convert_elapsed_ms > SLOW_THRESHOLD_MS { + slow_convert_count += 1; + if slow_convert_count <= 5 || slow_convert_count.is_multiple_of(100) { + debug!( + elapsed_ms = convert_elapsed_ms, + count = slow_convert_count, + "fill_frame_from_sample_buf exceeded {}ms threshold", + SLOW_THRESHOLD_MS + ); + } + } + + match fill_result { + Ok(()) => { + let encode_start = std::time::Instant::now(); + let owned_frame = frame_pool.take_frame(); + + if let Ok(mut encoder) = encoder_clone.lock() + && let Err(e) = encoder.queue_frame(owned_frame, timestamp) + { + warn!("Failed to encode frame: {e}"); + } + + let encode_elapsed_ms = encode_start.elapsed().as_millis(); + + if encode_elapsed_ms > SLOW_THRESHOLD_MS { + slow_encode_count += 1; + if slow_encode_count <= 5 || slow_encode_count.is_multiple_of(100) { + debug!( + elapsed_ms = encode_elapsed_ms, + count = slow_encode_count, + "encoder.queue_frame exceeded {}ms threshold", + SLOW_THRESHOLD_MS + ); + } + } + } + Err(e) => { + warn!("Failed to convert frame: {e:?}"); + } + } + + total_frames += 1; + } + + if total_frames > 0 { + debug!( + total_frames = total_frames, + slow_converts = slow_convert_count, + slow_encodes = slow_encode_count, + slow_convert_pct = format!( + "{:.1}%", + 100.0 * slow_convert_count as f64 / total_frames as f64 + ), + slow_encode_pct = format!( + "{:.1}%", + 100.0 * slow_encode_count as f64 / total_frames as f64 + ), + "M4S encoder timing summary (using SegmentedVideoEncoder)" + ); + } + + Ok(()) + })?; + + ready_rx + .recv() + .map_err(|_| anyhow!("M4S encoder thread ended unexpectedly"))??; + + self.state = Some(EncoderState { + video_tx, + encoder, + encoder_handle: Some(encoder_handle), + }); + + self.started = true; + + info!( + path = %self.base_path.display(), + "Started M4S fragmented video encoder" + ); + + Ok(()) + } +} + +impl VideoMuxer for MacOSFragmentedM4SMuxer { + type VideoFrame = screen_capture::VideoFrame; + + fn send_video_frame( + &mut self, + frame: Self::VideoFrame, + timestamp: Duration, + ) -> anyhow::Result<()> { + let Some(adjusted_timestamp) = self.pause.adjust(timestamp)? else { + return Ok(()); + }; + + if !self.started { + self.start_encoder()?; + } + + if let Some(state) = &self.state { + match state + .video_tx + .try_send(Some((frame.sample_buf, adjusted_timestamp))) + { + Ok(()) => { + self.frame_drops.record_frame(); + } + Err(e) => match e { + std::sync::mpsc::TrySendError::Full(_) => { + self.frame_drops.record_drop(); + } + std::sync::mpsc::TrySendError::Disconnected(_) => { + trace!("M4S encoder channel disconnected"); + } + }, + } + } + + Ok(()) + } +} + +impl AudioMuxer for MacOSFragmentedM4SMuxer { + fn send_audio_frame(&mut self, _frame: AudioFrame, _timestamp: Duration) -> anyhow::Result<()> { + Ok(()) + } +} + +fn copy_plane_data( + src: &[u8], + dest: &mut [u8], + height: usize, + row_width: usize, + src_stride: usize, + dest_stride: usize, +) { + if src_stride == row_width && dest_stride == row_width { + let total_bytes = height * row_width; + dest[..total_bytes].copy_from_slice(&src[..total_bytes]); + } else if src_stride == dest_stride { + let total_bytes = height * src_stride; + dest[..total_bytes].copy_from_slice(&src[..total_bytes]); + } else { + for y in 0..height { + let src_row = &src[y * src_stride..y * src_stride + row_width]; + let dest_row = &mut dest[y * dest_stride..y * dest_stride + row_width]; + dest_row.copy_from_slice(src_row); + } + } +} + +struct FramePool { + frame: Option, + pixel_format: ffmpeg::format::Pixel, + width: u32, + height: u32, +} + +impl FramePool { + fn new(pixel_format: ffmpeg::format::Pixel, width: u32, height: u32) -> Self { + Self { + frame: Some(ffmpeg::frame::Video::new(pixel_format, width, height)), + pixel_format, + width, + height, + } + } + + fn get_frame(&mut self) -> &mut ffmpeg::frame::Video { + if self.frame.is_none() { + self.frame = Some(ffmpeg::frame::Video::new( + self.pixel_format, + self.width, + self.height, + )); + } + self.frame.as_mut().expect("frame initialized above") + } + + fn take_frame(&mut self) -> ffmpeg::frame::Video { + self.frame.take().unwrap_or_else(|| { + ffmpeg::frame::Video::new(self.pixel_format, self.width, self.height) + }) + } +} + +fn fill_frame_from_sample_buf( + sample_buf: &cidre::cm::SampleBuf, + frame: &mut ffmpeg::frame::Video, +) -> Result<(), SampleBufConversionError> { + use cidre::cv::{self, pixel_buffer::LockFlags}; + + let Some(image_buf_ref) = sample_buf.image_buf() else { + return Err(SampleBufConversionError::NoImageBuffer); + }; + let mut image_buf = image_buf_ref.retained(); + + let width = image_buf.width(); + let height = image_buf.height(); + let pixel_format = image_buf.pixel_format(); + let plane0_stride = image_buf.plane_bytes_per_row(0); + let plane1_stride = image_buf.plane_bytes_per_row(1); + + let bytes_lock = BaseAddrLockGuard::lock(image_buf.as_mut(), LockFlags::READ_ONLY) + .map_err(SampleBufConversionError::BaseAddrLock)?; + + match pixel_format { + cv::PixelFormat::_420V => { + let dest_stride0 = frame.stride(0); + let dest_stride1 = frame.stride(1); + + copy_plane_data( + bytes_lock.plane_data(0), + frame.data_mut(0), + height, + width, + plane0_stride, + dest_stride0, + ); + + copy_plane_data( + bytes_lock.plane_data(1), + frame.data_mut(1), + height / 2, + width, + plane1_stride, + dest_stride1, + ); + } + cv::PixelFormat::_32_BGRA => { + let row_width = width * 4; + let dest_stride = frame.stride(0); + copy_plane_data( + bytes_lock.plane_data(0), + frame.data_mut(0), + height, + row_width, + plane0_stride, + dest_stride, + ); + } + cv::PixelFormat::_2VUY => { + let row_width = width * 2; + let dest_stride = frame.stride(0); + copy_plane_data( + bytes_lock.plane_data(0), + frame.data_mut(0), + height, + row_width, + plane0_stride, + dest_stride, + ); + } + format => return Err(SampleBufConversionError::UnsupportedFormat(format)), + } + + Ok(()) +} + +#[derive(Debug)] +#[allow(dead_code)] +enum SampleBufConversionError { + UnsupportedFormat(cidre::cv::PixelFormat), + BaseAddrLock(cidre::os::Error), + NoImageBuffer, +} + +struct BaseAddrLockGuard<'a>( + &'a mut cidre::cv::ImageBuf, + cidre::cv::pixel_buffer::LockFlags, +); + +impl<'a> BaseAddrLockGuard<'a> { + fn lock( + image_buf: &'a mut cidre::cv::ImageBuf, + flags: cidre::cv::pixel_buffer::LockFlags, + ) -> cidre::os::Result { + unsafe { image_buf.lock_base_addr(flags) }.result()?; + Ok(Self(image_buf, flags)) + } + + fn plane_data(&self, index: usize) -> &[u8] { + let base_addr = self.0.plane_base_address(index); + let plane_size = self.0.plane_bytes_per_row(index); + unsafe { std::slice::from_raw_parts(base_addr, plane_size * self.0.plane_height(index)) } + } +} + +impl Drop for BaseAddrLockGuard<'_> { + fn drop(&mut self) { + unsafe { self.0.unlock_lock_base_addr(self.1) }; + } +} + +pub struct MacOSFragmentedM4SCameraMuxer { + base_path: PathBuf, + video_config: VideoInfo, + segment_duration: Duration, + preset: H264Preset, + output_size: Option<(u32, u32)>, + state: Option, + pause: SharedPauseState, + frame_drops: FrameDropTracker, + started: bool, +} + +pub struct MacOSFragmentedM4SCameraMuxerConfig { + pub segment_duration: Duration, + pub preset: H264Preset, + pub output_size: Option<(u32, u32)>, + pub shared_pause_state: Option, +} + +impl Default for MacOSFragmentedM4SCameraMuxerConfig { + fn default() -> Self { + Self { + segment_duration: Duration::from_secs(3), + preset: H264Preset::Ultrafast, + output_size: None, + shared_pause_state: None, + } + } +} + +impl Muxer for MacOSFragmentedM4SCameraMuxer { + type Config = MacOSFragmentedM4SCameraMuxerConfig; + + async fn setup( + config: Self::Config, + output_path: PathBuf, + video_config: Option, + _audio_config: Option, + pause_flag: Arc, + _tasks: &mut TaskPool, + ) -> anyhow::Result + where + Self: Sized, + { + let video_config = + video_config.ok_or_else(|| anyhow!("invariant: video config expected for camera"))?; + + std::fs::create_dir_all(&output_path).with_context(|| { + format!("Failed to create camera segments directory: {output_path:?}") + })?; + + let pause = config + .shared_pause_state + .unwrap_or_else(|| SharedPauseState::new(pause_flag)); + + Ok(Self { + base_path: output_path, + video_config, + segment_duration: config.segment_duration, + preset: config.preset, + output_size: config.output_size, + state: None, + pause, + frame_drops: FrameDropTracker::new(), + started: false, + }) + } + + fn stop(&mut self) { + if let Some(state) = &self.state + && let Err(e) = state.video_tx.send(None) + { + trace!("M4S camera encoder channel already closed during stop: {e}"); + } + } + + fn finish(&mut self, timestamp: Duration) -> anyhow::Result> { + if let Some(mut state) = self.state.take() { + if let Err(e) = state.video_tx.send(None) { + trace!("M4S camera encoder channel already closed during finish: {e}"); + } + + if let Some(handle) = state.encoder_handle.take() { + let timeout = Duration::from_secs(5); + let start = std::time::Instant::now(); + loop { + if handle.is_finished() { + match handle.join() { + Err(panic_payload) => { + warn!( + "M4S camera encoder thread panicked during finish: {:?}", + panic_payload + ); + } + Ok(Err(e)) => { + warn!("M4S camera encoder thread returned error: {e}"); + } + Ok(Ok(())) => {} + } + break; + } + if start.elapsed() > timeout { + warn!( + "M4S camera encoder thread did not finish within {:?}, abandoning", + timeout + ); + break; + } + std::thread::sleep(Duration::from_millis(50)); + } + } + + if let Ok(mut encoder) = state.encoder.lock() + && let Err(e) = encoder.finish_with_timestamp(timestamp) + { + warn!("Failed to finish camera segmented encoder: {e}"); + } + } + + Ok(Ok(())) + } +} + +impl MacOSFragmentedM4SCameraMuxer { + fn start_encoder(&mut self) -> anyhow::Result<()> { + let buffer_size = get_muxer_buffer_size(); + debug!( + buffer_size = buffer_size, + "M4S camera muxer encoder channel buffer size" + ); + + let (video_tx, video_rx) = + sync_channel::, Duration)>>(buffer_size); + let (ready_tx, ready_rx) = sync_channel::>(1); + + let encoder_config = SegmentedVideoEncoderConfig { + segment_duration: self.segment_duration, + preset: self.preset, + bpp: H264EncoderBuilder::QUALITY_BPP, + output_size: self.output_size, + }; + + let encoder = + SegmentedVideoEncoder::init(self.base_path.clone(), self.video_config, encoder_config)?; + let encoder = Arc::new(Mutex::new(encoder)); + let encoder_clone = encoder.clone(); + let video_config = self.video_config; + + let encoder_handle = std::thread::Builder::new() + .name("m4s-camera-segment-encoder".to_string()) + .spawn(move || { + let pixel_format = match video_config.pixel_format { + cap_media_info::Pixel::NV12 => ffmpeg::format::Pixel::NV12, + cap_media_info::Pixel::BGRA => ffmpeg::format::Pixel::BGRA, + cap_media_info::Pixel::UYVY422 => ffmpeg::format::Pixel::UYVY422, + _ => ffmpeg::format::Pixel::NV12, + }; + + let mut frame_pool = + FramePool::new(pixel_format, video_config.width, video_config.height); + + if ready_tx.send(Ok(())).is_err() { + return Err(anyhow!( + "Failed to send ready signal - camera receiver dropped" + )); + } + + let mut slow_convert_count = 0u32; + let mut slow_encode_count = 0u32; + let mut total_frames = 0u64; + const SLOW_THRESHOLD_MS: u128 = 5; + + while let Ok(Some((sample_buf, timestamp))) = video_rx.recv() { + let convert_start = std::time::Instant::now(); + let frame = frame_pool.get_frame(); + let fill_result = fill_frame_from_sample_buf(&sample_buf, frame); + let convert_elapsed_ms = convert_start.elapsed().as_millis(); + + if convert_elapsed_ms > SLOW_THRESHOLD_MS { + slow_convert_count += 1; + if slow_convert_count <= 5 || slow_convert_count.is_multiple_of(100) { + debug!( + elapsed_ms = convert_elapsed_ms, + count = slow_convert_count, + "Camera fill_frame_from_sample_buf exceeded {}ms threshold", + SLOW_THRESHOLD_MS + ); + } + } + + match fill_result { + Ok(()) => { + let encode_start = std::time::Instant::now(); + let owned_frame = frame_pool.take_frame(); + + if let Ok(mut encoder) = encoder_clone.lock() + && let Err(e) = encoder.queue_frame(owned_frame, timestamp) + { + warn!("Failed to encode camera frame: {e}"); + } + + let encode_elapsed_ms = encode_start.elapsed().as_millis(); + + if encode_elapsed_ms > SLOW_THRESHOLD_MS { + slow_encode_count += 1; + if slow_encode_count <= 5 || slow_encode_count.is_multiple_of(100) { + debug!( + elapsed_ms = encode_elapsed_ms, + count = slow_encode_count, + "Camera encoder.queue_frame exceeded {}ms threshold", + SLOW_THRESHOLD_MS + ); + } + } + } + Err(e) => { + warn!("Failed to convert camera frame: {e:?}"); + } + } + + total_frames += 1; + } + + if total_frames > 0 { + debug!( + total_frames = total_frames, + slow_converts = slow_convert_count, + slow_encodes = slow_encode_count, + slow_convert_pct = format!( + "{:.1}%", + 100.0 * slow_convert_count as f64 / total_frames as f64 + ), + slow_encode_pct = format!( + "{:.1}%", + 100.0 * slow_encode_count as f64 / total_frames as f64 + ), + "M4S camera encoder timing summary" + ); + } + + Ok(()) + })?; + + ready_rx + .recv() + .map_err(|_| anyhow!("M4S camera encoder thread ended unexpectedly"))??; + + self.state = Some(EncoderState { + video_tx, + encoder, + encoder_handle: Some(encoder_handle), + }); + + self.started = true; + + info!( + path = %self.base_path.display(), + "Started M4S fragmented camera encoder" + ); + + Ok(()) + } +} + +impl VideoMuxer for MacOSFragmentedM4SCameraMuxer { + type VideoFrame = NativeCameraFrame; + + fn send_video_frame( + &mut self, + frame: Self::VideoFrame, + timestamp: Duration, + ) -> anyhow::Result<()> { + let Some(adjusted_timestamp) = self.pause.adjust(timestamp)? else { + return Ok(()); + }; + + if !self.started { + self.start_encoder()?; + } + + if let Some(state) = &self.state { + match state + .video_tx + .try_send(Some((frame.sample_buf, adjusted_timestamp))) + { + Ok(()) => { + self.frame_drops.record_frame(); + } + Err(e) => match e { + std::sync::mpsc::TrySendError::Full(_) => { + self.frame_drops.record_drop(); + } + std::sync::mpsc::TrySendError::Disconnected(_) => { + trace!("M4S camera encoder channel disconnected"); + } + }, + } + } + + Ok(()) + } +} + +impl AudioMuxer for MacOSFragmentedM4SCameraMuxer { + fn send_audio_frame(&mut self, _frame: AudioFrame, _timestamp: Duration) -> anyhow::Result<()> { + Ok(()) + } +} diff --git a/crates/recording/src/output_pipeline/macos_segmented_ffmpeg.rs b/crates/recording/src/output_pipeline/macos_segmented_ffmpeg.rs deleted file mode 100644 index ad128644d9..0000000000 --- a/crates/recording/src/output_pipeline/macos_segmented_ffmpeg.rs +++ /dev/null @@ -1,746 +0,0 @@ -use crate::{AudioFrame, AudioMuxer, Muxer, TaskPool, VideoMuxer, fragmentation, screen_capture}; -use anyhow::{Context, anyhow}; -use cap_media_info::{AudioInfo, VideoInfo}; -use serde::Serialize; -use std::{ - path::PathBuf, - sync::{ - Arc, Mutex, - atomic::{AtomicBool, Ordering}, - mpsc::{SyncSender, sync_channel}, - }, - thread::JoinHandle, - time::Duration, -}; -use tracing::*; - -#[derive(Debug, Clone)] -pub struct SegmentInfo { - pub path: PathBuf, - pub index: u32, - pub duration: Duration, - pub file_size: Option, -} - -#[derive(Serialize)] -struct FragmentEntry { - path: String, - index: u32, - duration: f64, - is_complete: bool, - #[serde(skip_serializing_if = "Option::is_none")] - file_size: Option, -} - -const MANIFEST_VERSION: u32 = 2; - -#[derive(Serialize)] -struct Manifest { - version: u32, - fragments: Vec, - #[serde(skip_serializing_if = "Option::is_none")] - total_duration: Option, - is_complete: bool, -} - -struct SegmentState { - video_tx: SyncSender, Duration)>>, - output: Arc>, - encoder_handle: Option>>, -} - -struct PauseTracker { - flag: Arc, - paused_at: Option, - offset: Duration, -} - -struct FrameDropTracker { - count: u32, - last_warning: std::time::Instant, -} - -impl FrameDropTracker { - fn new() -> Self { - Self { - count: 0, - last_warning: std::time::Instant::now(), - } - } - - fn record_drop(&mut self) { - self.count += 1; - if self.count >= 30 && self.last_warning.elapsed() > Duration::from_secs(5) { - warn!( - "Dropped {} screen frames due to encoder backpressure", - self.count - ); - self.count = 0; - self.last_warning = std::time::Instant::now(); - } - } - - fn reset(&mut self) { - if self.count > 0 { - trace!("Frame drop count at segment boundary: {}", self.count); - } - self.count = 0; - } -} - -impl PauseTracker { - fn new(flag: Arc) -> Self { - Self { - flag, - paused_at: None, - offset: Duration::ZERO, - } - } - - fn adjust(&mut self, timestamp: Duration) -> anyhow::Result> { - if self.flag.load(Ordering::Relaxed) { - if self.paused_at.is_none() { - self.paused_at = Some(timestamp); - } - return Ok(None); - } - - if let Some(start) = self.paused_at.take() { - let delta = timestamp.checked_sub(start).ok_or_else(|| { - anyhow!( - "Frame timestamp went backward during unpause (resume={start:?}, current={timestamp:?})" - ) - })?; - - self.offset = self.offset.checked_add(delta).ok_or_else(|| { - anyhow!( - "Pause offset overflow (offset={:?}, delta={delta:?})", - self.offset - ) - })?; - } - - let adjusted = timestamp.checked_sub(self.offset).ok_or_else(|| { - anyhow!( - "Adjusted timestamp underflow (timestamp={timestamp:?}, offset={:?})", - self.offset - ) - })?; - - Ok(Some(adjusted)) - } -} - -pub struct MacOSSegmentedMuxer { - base_path: PathBuf, - segment_duration: Duration, - current_index: u32, - segment_start_time: Option, - completed_segments: Vec, - pending_segments: Arc>>, - - current_state: Option, - - video_config: VideoInfo, - - pause: PauseTracker, - frame_drops: FrameDropTracker, -} - -pub struct MacOSSegmentedMuxerConfig { - pub segment_duration: Duration, -} - -impl Default for MacOSSegmentedMuxerConfig { - fn default() -> Self { - Self { - segment_duration: Duration::from_secs(3), - } - } -} - -impl Muxer for MacOSSegmentedMuxer { - type Config = MacOSSegmentedMuxerConfig; - - async fn setup( - config: Self::Config, - output_path: PathBuf, - video_config: Option, - _audio_config: Option, - pause_flag: Arc, - _tasks: &mut TaskPool, - ) -> anyhow::Result - where - Self: Sized, - { - let video_config = - video_config.ok_or_else(|| anyhow!("invariant: video config expected"))?; - - std::fs::create_dir_all(&output_path) - .with_context(|| format!("Failed to create segments directory: {output_path:?}"))?; - - Ok(Self { - base_path: output_path, - segment_duration: config.segment_duration, - current_index: 0, - segment_start_time: None, - completed_segments: Vec::new(), - pending_segments: Arc::new(Mutex::new(Vec::new())), - current_state: None, - video_config, - pause: PauseTracker::new(pause_flag), - frame_drops: FrameDropTracker::new(), - }) - } - - fn stop(&mut self) { - if let Some(state) = &self.current_state - && let Err(e) = state.video_tx.send(None) - { - trace!("Screen encoder channel already closed during stop: {e}"); - } - } - - fn finish(&mut self, timestamp: Duration) -> anyhow::Result> { - self.collect_pending_segments(); - - let segment_path = self.current_segment_path(); - let segment_start = self.segment_start_time; - let current_index = self.current_index; - - if let Some(mut state) = self.current_state.take() { - if let Err(e) = state.video_tx.send(None) { - trace!("Screen encoder channel already closed during finish: {e}"); - } - - if let Some(handle) = state.encoder_handle.take() { - let timeout = Duration::from_secs(5); - let start = std::time::Instant::now(); - loop { - if handle.is_finished() { - if let Err(panic_payload) = handle.join() { - warn!( - "Screen encoder thread panicked during finish: {:?}", - panic_payload - ); - } - break; - } - if start.elapsed() > timeout { - warn!( - "Screen encoder thread did not finish within {:?}, abandoning", - timeout - ); - break; - } - std::thread::sleep(Duration::from_millis(50)); - } - } - - if let Ok(mut output) = state.output.lock() - && let Err(e) = output.write_trailer() - { - warn!("Failed to write trailer for segment {current_index}: {e}"); - } - - fragmentation::sync_file(&segment_path); - - if let Some(start) = segment_start { - let final_duration = timestamp.saturating_sub(start); - let file_size = std::fs::metadata(&segment_path).ok().map(|m| m.len()); - - self.completed_segments.push(SegmentInfo { - path: segment_path, - index: current_index, - duration: final_duration, - file_size, - }); - } - } - - self.finalize_manifest(); - - Ok(Ok(())) - } -} - -impl MacOSSegmentedMuxer { - fn current_segment_path(&self) -> PathBuf { - self.base_path - .join(format!("fragment_{:03}.mp4", self.current_index)) - } - - fn write_manifest(&self) { - let manifest = Manifest { - version: MANIFEST_VERSION, - fragments: self - .completed_segments - .iter() - .map(|s| FragmentEntry { - path: s - .path - .file_name() - .unwrap_or_default() - .to_string_lossy() - .into_owned(), - index: s.index, - duration: s.duration.as_secs_f64(), - is_complete: true, - file_size: s.file_size, - }) - .collect(), - total_duration: None, - is_complete: false, - }; - - let manifest_path = self.base_path.join("manifest.json"); - if let Err(e) = fragmentation::atomic_write_json(&manifest_path, &manifest) { - warn!( - "Failed to write manifest to {}: {e}", - manifest_path.display() - ); - } - } - - fn finalize_manifest(&self) { - let total_duration: Duration = self.completed_segments.iter().map(|s| s.duration).sum(); - - let manifest = Manifest { - version: MANIFEST_VERSION, - fragments: self - .completed_segments - .iter() - .map(|s| FragmentEntry { - path: s - .path - .file_name() - .unwrap_or_default() - .to_string_lossy() - .into_owned(), - index: s.index, - duration: s.duration.as_secs_f64(), - is_complete: true, - file_size: s.file_size, - }) - .collect(), - total_duration: Some(total_duration.as_secs_f64()), - is_complete: true, - }; - - let manifest_path = self.base_path.join("manifest.json"); - if let Err(e) = fragmentation::atomic_write_json(&manifest_path, &manifest) { - warn!( - "Failed to write final manifest to {}: {e}", - manifest_path.display() - ); - } - } - - fn collect_pending_segments(&mut self) { - if let Ok(mut pending) = self.pending_segments.lock() { - for segment in pending.drain(..) { - self.completed_segments.push(segment); - } - self.completed_segments.sort_by_key(|s| s.index); - } - } - - fn create_segment(&mut self) -> anyhow::Result<()> { - let segment_path = self.current_segment_path(); - - let (video_tx, video_rx) = - sync_channel::, Duration)>>(8); - let (ready_tx, ready_rx) = sync_channel::>(1); - let output = ffmpeg::format::output(&segment_path)?; - let output = Arc::new(Mutex::new(output)); - - let video_config = self.video_config; - let output_clone = output.clone(); - - let encoder_handle = std::thread::Builder::new() - .name(format!("segment-encoder-{}", self.current_index)) - .spawn(move || { - let encoder = (|| { - let mut output_guard = match output_clone.lock() { - Ok(guard) => guard, - Err(poisoned) => { - return Err(anyhow!( - "MacOSSegmentedEncoder: failed to lock output mutex: {}", - poisoned - )); - } - }; - - cap_enc_ffmpeg::h264::H264Encoder::builder(video_config) - .build(&mut output_guard) - .map_err(|e| anyhow!("MacOSSegmentedEncoder/{e}")) - })(); - - let mut encoder = match encoder { - Ok(encoder) => { - if ready_tx.send(Ok(())).is_err() { - error!("Failed to send ready signal - receiver dropped"); - return Ok(()); - } - encoder - } - Err(e) => { - error!("Encoder setup failed: {:#}", e); - if let Err(send_err) = ready_tx.send(Err(anyhow!("{e}"))) { - error!("failed to send ready_tx error: {send_err}"); - } - return Err(anyhow!("{e}")); - } - }; - - let mut first_timestamp: Option = None; - - while let Ok(Some((sample_buf, timestamp))) = video_rx.recv() { - let Ok(mut output) = output_clone.lock() else { - continue; - }; - - let relative = if let Some(first) = first_timestamp { - timestamp.checked_sub(first).unwrap_or(Duration::ZERO) - } else { - first_timestamp = Some(timestamp); - Duration::ZERO - }; - - let frame = sample_buf_to_ffmpeg_frame(&sample_buf); - - match frame { - Ok(frame) => { - if let Err(e) = encoder.queue_frame(frame, relative, &mut output) { - warn!("Failed to encode frame: {e}"); - } - } - Err(e) => { - warn!("Failed to convert frame: {e:?}"); - } - } - } - - if let Ok(mut output) = output_clone.lock() - && let Err(e) = encoder.flush(&mut output) - { - warn!("Failed to flush encoder: {e}"); - } - - drop(encoder); - - Ok(()) - })?; - - ready_rx - .recv() - .map_err(|_| anyhow!("Encoder thread ended unexpectedly"))??; - - output - .lock() - .map_err(|_| anyhow!("output mutex poisoned when writing header"))? - .write_header()?; - - self.current_state = Some(SegmentState { - video_tx, - output, - encoder_handle: Some(encoder_handle), - }); - - Ok(()) - } - - fn rotate_segment(&mut self, timestamp: Duration) -> anyhow::Result<()> { - self.collect_pending_segments(); - - let segment_start = self.segment_start_time.unwrap_or(Duration::ZERO); - let segment_duration = timestamp.saturating_sub(segment_start); - let completed_segment_path = self.current_segment_path(); - let current_index = self.current_index; - - if let Some(mut state) = self.current_state.take() { - if let Err(e) = state.video_tx.send(None) { - trace!("Screen encoder channel already closed during rotation: {e}"); - } - - let output = state.output.clone(); - let encoder_handle = state.encoder_handle.take(); - let path_for_sync = completed_segment_path.clone(); - let pending_segments = self.pending_segments.clone(); - - std::thread::spawn(move || { - if let Some(handle) = encoder_handle { - let timeout = Duration::from_secs(5); - let start = std::time::Instant::now(); - loop { - if handle.is_finished() { - if let Err(panic_payload) = handle.join() { - warn!( - "Screen encoder thread panicked during rotation: {:?}", - panic_payload - ); - } - break; - } - if start.elapsed() > timeout { - warn!( - "Screen encoder thread did not finish within {:?} during rotation, abandoning", - timeout - ); - break; - } - std::thread::sleep(Duration::from_millis(50)); - } - } - - if let Ok(mut output) = output.lock() - && let Err(e) = output.write_trailer() - { - warn!("Failed to write trailer for segment {current_index}: {e}"); - } - - fragmentation::sync_file(&path_for_sync); - - let file_size = std::fs::metadata(&path_for_sync).ok().map(|m| m.len()); - - if let Ok(mut pending) = pending_segments.lock() { - pending.push(SegmentInfo { - path: path_for_sync, - index: current_index, - duration: segment_duration, - file_size, - }); - } - }); - - self.write_manifest(); - } - - self.frame_drops.reset(); - self.current_index += 1; - self.segment_start_time = Some(timestamp); - - self.create_segment()?; - self.write_in_progress_manifest(); - - info!( - "Rotated to segment {} at {:?}", - self.current_index, timestamp - ); - - Ok(()) - } - - fn write_in_progress_manifest(&self) { - let mut fragments: Vec = self - .completed_segments - .iter() - .map(|s| FragmentEntry { - path: s - .path - .file_name() - .unwrap_or_default() - .to_string_lossy() - .into_owned(), - index: s.index, - duration: s.duration.as_secs_f64(), - is_complete: true, - file_size: s.file_size, - }) - .collect(); - - fragments.push(FragmentEntry { - path: self - .current_segment_path() - .file_name() - .unwrap_or_default() - .to_string_lossy() - .into_owned(), - index: self.current_index, - duration: 0.0, - is_complete: false, - file_size: None, - }); - - let manifest = Manifest { - version: MANIFEST_VERSION, - fragments, - total_duration: None, - is_complete: false, - }; - - let manifest_path = self.base_path.join("manifest.json"); - if let Err(e) = fragmentation::atomic_write_json(&manifest_path, &manifest) { - warn!( - "Failed to write in-progress manifest to {}: {e}", - manifest_path.display() - ); - } - } -} - -impl VideoMuxer for MacOSSegmentedMuxer { - type VideoFrame = screen_capture::VideoFrame; - - fn send_video_frame( - &mut self, - frame: Self::VideoFrame, - timestamp: Duration, - ) -> anyhow::Result<()> { - let Some(adjusted_timestamp) = self.pause.adjust(timestamp)? else { - return Ok(()); - }; - - if self.current_state.is_none() { - self.segment_start_time = Some(adjusted_timestamp); - self.create_segment()?; - self.write_in_progress_manifest(); - } - - if self.segment_start_time.is_none() { - self.segment_start_time = Some(adjusted_timestamp); - } - - let segment_elapsed = - adjusted_timestamp.saturating_sub(self.segment_start_time.unwrap_or(Duration::ZERO)); - - if segment_elapsed >= self.segment_duration { - self.rotate_segment(adjusted_timestamp)?; - } - - if let Some(state) = &self.current_state - && let Err(e) = state - .video_tx - .try_send(Some((frame.sample_buf, adjusted_timestamp))) - { - match e { - std::sync::mpsc::TrySendError::Full(_) => { - self.frame_drops.record_drop(); - } - std::sync::mpsc::TrySendError::Disconnected(_) => { - trace!("Screen encoder channel disconnected"); - } - } - } - - Ok(()) - } -} - -impl AudioMuxer for MacOSSegmentedMuxer { - fn send_audio_frame(&mut self, _frame: AudioFrame, _timestamp: Duration) -> anyhow::Result<()> { - Ok(()) - } -} - -fn sample_buf_to_ffmpeg_frame( - sample_buf: &cidre::cm::SampleBuf, -) -> Result { - use cidre::cv::{self, pixel_buffer::LockFlags}; - - let Some(image_buf_ref) = sample_buf.image_buf() else { - return Err(SampleBufConversionError::NoImageBuffer); - }; - let mut image_buf = image_buf_ref.retained(); - - let width = image_buf.width(); - let height = image_buf.height(); - let pixel_format = image_buf.pixel_format(); - let plane0_stride = image_buf.plane_bytes_per_row(0); - let plane1_stride = image_buf.plane_bytes_per_row(1); - - let bytes_lock = BaseAddrLockGuard::lock(image_buf.as_mut(), LockFlags::READ_ONLY) - .map_err(SampleBufConversionError::BaseAddrLock)?; - - Ok(match pixel_format { - cv::PixelFormat::_420V => { - let mut ff_frame = - ffmpeg::frame::Video::new(ffmpeg::format::Pixel::NV12, width as u32, height as u32); - - let src_stride = plane0_stride; - let dest_stride = ff_frame.stride(0); - - let src_bytes = bytes_lock.plane_data(0); - let dest_bytes = &mut ff_frame.data_mut(0); - - for y in 0..height { - let row_width = width; - let src_row = &src_bytes[y * src_stride..y * src_stride + row_width]; - let dest_row = &mut dest_bytes[y * dest_stride..y * dest_stride + row_width]; - - dest_row.copy_from_slice(src_row); - } - - let src_stride = plane1_stride; - let dest_stride = ff_frame.stride(1); - - let src_bytes = bytes_lock.plane_data(1); - let dest_bytes = &mut ff_frame.data_mut(1); - - for y in 0..height / 2 { - let row_width = width; - let src_row = &src_bytes[y * src_stride..y * src_stride + row_width]; - let dest_row = &mut dest_bytes[y * dest_stride..y * dest_stride + row_width]; - - dest_row.copy_from_slice(src_row); - } - - ff_frame - } - cv::PixelFormat::_32_BGRA => { - let mut ff_frame = - ffmpeg::frame::Video::new(ffmpeg::format::Pixel::BGRA, width as u32, height as u32); - - let src_stride = plane0_stride; - let dest_stride = ff_frame.stride(0); - - let src_bytes = bytes_lock.plane_data(0); - let dest_bytes = &mut ff_frame.data_mut(0); - - for y in 0..height { - let row_width = width * 4; - let src_row = &src_bytes[y * src_stride..y * src_stride + row_width]; - let dest_row = &mut dest_bytes[y * dest_stride..y * dest_stride + row_width]; - - dest_row.copy_from_slice(src_row); - } - - ff_frame - } - format => return Err(SampleBufConversionError::UnsupportedFormat(format)), - }) -} - -#[derive(Debug)] -pub enum SampleBufConversionError { - UnsupportedFormat(cidre::cv::PixelFormat), - BaseAddrLock(cidre::os::Error), - NoImageBuffer, -} - -struct BaseAddrLockGuard<'a>( - &'a mut cidre::cv::ImageBuf, - cidre::cv::pixel_buffer::LockFlags, -); - -impl<'a> BaseAddrLockGuard<'a> { - fn lock( - image_buf: &'a mut cidre::cv::ImageBuf, - flags: cidre::cv::pixel_buffer::LockFlags, - ) -> cidre::os::Result { - unsafe { image_buf.lock_base_addr(flags) }.result()?; - Ok(Self(image_buf, flags)) - } - - fn plane_data(&self, index: usize) -> &[u8] { - let base_addr = self.0.plane_base_address(index); - let plane_size = self.0.plane_bytes_per_row(index); - unsafe { std::slice::from_raw_parts(base_addr, plane_size * self.0.plane_height(index)) } - } -} - -impl Drop for BaseAddrLockGuard<'_> { - fn drop(&mut self) { - let _ = unsafe { self.0.unlock_lock_base_addr(self.1) }; - } -} diff --git a/crates/recording/src/output_pipeline/mod.rs b/crates/recording/src/output_pipeline/mod.rs index bf1df859ef..2f2bf6d2f1 100644 --- a/crates/recording/src/output_pipeline/mod.rs +++ b/crates/recording/src/output_pipeline/mod.rs @@ -2,17 +2,13 @@ mod async_camera; mod core; pub mod ffmpeg; #[cfg(target_os = "macos")] -mod fragmented; -#[cfg(target_os = "macos")] -mod macos_segmented_ffmpeg; +mod macos_fragmented_m4s; pub use async_camera::*; pub use core::*; pub use ffmpeg::*; #[cfg(target_os = "macos")] -pub use fragmented::*; -#[cfg(target_os = "macos")] -pub use macos_segmented_ffmpeg::*; +pub use macos_fragmented_m4s::*; #[cfg(target_os = "macos")] mod macos; diff --git a/crates/recording/src/recovery.rs b/crates/recording/src/recovery.rs index 4490ac1c53..63f4ba4ed6 100644 --- a/crates/recording/src/recovery.rs +++ b/crates/recording/src/recovery.rs @@ -4,8 +4,9 @@ use std::{ }; use cap_enc_ffmpeg::remux::{ - concatenate_audio_to_ogg, concatenate_video_fragments, get_media_duration, get_video_fps, - probe_media_valid, probe_video_can_decode, + concatenate_audio_to_ogg, concatenate_m4s_segments_with_init, concatenate_video_fragments, + get_media_duration, get_video_fps, probe_m4s_can_decode_with_init, probe_media_valid, + probe_video_can_decode, }; use cap_project::{ AudioMeta, Cursors, MultipleSegment, MultipleSegments, ProjectConfiguration, RecordingMeta, @@ -27,7 +28,9 @@ pub struct IncompleteRecording { pub struct RecoverableSegment { pub index: u32, pub display_fragments: Vec, + pub display_init_segment: Option, pub camera_fragments: Option>, + pub camera_init_segment: Option, pub mic_fragments: Option>, pub system_audio_fragments: Option>, pub cursor_path: Option, @@ -39,6 +42,12 @@ pub struct RecoveredRecording { pub meta: StudioRecordingMeta, } +#[derive(Debug, Clone)] +struct FragmentsInfo { + fragments: Vec, + init_segment: Option, +} + #[derive(Debug, thiserror::Error)] pub enum RecoveryError { #[error("IO error: {0}")] @@ -133,13 +142,16 @@ impl RecoveryManager { let segment_path = segment_entry.path(); let display_dir = segment_path.join("display"); - let mut display_fragments = Self::find_complete_fragments(&display_dir); + let display_info = Self::find_complete_fragments_with_init(&display_dir); + let mut display_fragments = display_info.fragments; + let mut display_init_segment = display_info.init_segment; if display_fragments.is_empty() && let Some(display_mp4) = Self::probe_single_file(&segment_path.join("display.mp4")) { display_fragments = vec![display_mp4]; + display_init_segment = None; } if display_fragments.is_empty() { @@ -151,12 +163,15 @@ impl RecoveryManager { } let camera_dir = segment_path.join("camera"); - let camera_fragments = { - let frags = Self::find_complete_fragments(&camera_dir); - if frags.is_empty() { - Self::probe_single_file(&segment_path.join("camera.mp4")).map(|p| vec![p]) + let (camera_fragments, camera_init_segment) = { + let camera_info = Self::find_complete_fragments_with_init(&camera_dir); + if camera_info.fragments.is_empty() { + ( + Self::probe_single_file(&segment_path.join("camera.mp4")).map(|p| vec![p]), + None, + ) } else { - Some(frags) + (Some(camera_info.fragments), camera_info.init_segment) } }; @@ -173,7 +188,9 @@ impl RecoveryManager { recoverable_segments.push(RecoverableSegment { index: index as u32, display_fragments, + display_init_segment, camera_fragments, + camera_init_segment, mic_fragments, system_audio_fragments, cursor_path, @@ -201,6 +218,10 @@ impl RecoveryManager { } fn find_complete_fragments(dir: &Path) -> Vec { + Self::find_complete_fragments_with_init(dir).fragments + } + + fn find_complete_fragments_with_init(dir: &Path) -> FragmentsInfo { use crate::fragmentation::CURRENT_MANIFEST_VERSION; let manifest_path = dir.join("manifest.json"); @@ -208,82 +229,141 @@ impl RecoveryManager { if manifest_path.exists() && let Ok(content) = std::fs::read_to_string(&manifest_path) && let Ok(manifest) = serde_json::from_str::(&content) - && let Some(fragments) = manifest.get("fragments").and_then(|f| f.as_array()) { let manifest_version = manifest .get("version") .and_then(|v| v.as_u64()) .unwrap_or(1) as u32; - if manifest_version > CURRENT_MANIFEST_VERSION { + + let manifest_type = manifest + .get("type") + .and_then(|t| t.as_str()) + .unwrap_or("fragments"); + + let max_supported_version = if manifest_type == "m4s_segments" { + 4 + } else { + CURRENT_MANIFEST_VERSION + }; + + if manifest_version > max_supported_version { warn!( - "Manifest version {} is newer than supported {}", - manifest_version, CURRENT_MANIFEST_VERSION + "Manifest version {} is newer than supported {} for type {}", + manifest_version, max_supported_version, manifest_type ); } - let expected_file_size = |f: &serde_json::Value| -> Option { - f.get("file_size").and_then(|s| s.as_u64()) + let init_segment = manifest + .get("init_segment") + .and_then(|i| i.as_str()) + .map(|name| dir.join(name)) + .filter(|p| p.exists()); + + let entries = if manifest_type == "m4s_segments" { + manifest.get("segments").and_then(|s| s.as_array()) + } else { + manifest.get("fragments").and_then(|f| f.as_array()) }; - let result: Vec = fragments - .iter() - .filter(|f| { - f.get("is_complete") - .and_then(|c| c.as_bool()) - .unwrap_or(false) - }) - .filter_map(|f| { - let path_str = f.get("path").and_then(|p| p.as_str())?; - let path = dir.join(path_str); - if !path.exists() { - return None; - } + if let Some(entries) = entries { + let expected_file_size = |f: &serde_json::Value| -> Option { + f.get("file_size").and_then(|s| s.as_u64()) + }; + + let result: Vec = entries + .iter() + .filter(|f| { + f.get("is_complete") + .and_then(|c| c.as_bool()) + .unwrap_or(false) + }) + .filter_map(|f| { + let path_str = f.get("path").and_then(|p| p.as_str())?; + let path = dir.join(path_str); + if !path.exists() { + return None; + } - if let Some(expected_size) = expected_file_size(f) - && let Ok(metadata) = std::fs::metadata(&path) - && metadata.len() != expected_size - { - warn!( - "Fragment {} size mismatch: expected {}, got {}", - path.display(), - expected_size, - metadata.len() - ); - return None; - } + if let Some(expected_size) = expected_file_size(f) + && let Ok(metadata) = std::fs::metadata(&path) + && metadata.len() != expected_size + { + warn!( + "Fragment {} size mismatch: expected {}, got {}", + path.display(), + expected_size, + metadata.len() + ); + return None; + } - if Self::is_video_file(&path) { - match probe_video_can_decode(&path) { - Ok(true) => Some(path), - Ok(false) => { - warn!("Fragment {} has no decodable frames", path.display()); - None - } - Err(e) => { - warn!("Fragment {} validation failed: {}", path.display(), e); - None + if Self::is_video_file(&path) { + if let Some(init_path) = &init_segment { + match probe_m4s_can_decode_with_init(init_path, &path) { + Ok(true) => Some(path), + Ok(false) => { + warn!( + "M4S segment {} has no decodable frames (with init)", + path.display() + ); + None + } + Err(e) => { + warn!( + "M4S segment {} validation failed: {}", + path.display(), + e + ); + None + } + } + } else { + match probe_video_can_decode(&path) { + Ok(true) => Some(path), + Ok(false) => { + warn!( + "Fragment {} has no decodable frames", + path.display() + ); + None + } + Err(e) => { + warn!( + "Fragment {} validation failed: {}", + path.display(), + e + ); + None + } + } } + } else if probe_media_valid(&path) { + Some(path) + } else { + warn!("Fragment {} is not valid media", path.display()); + None } - } else if probe_media_valid(&path) { - Some(path) - } else { - warn!("Fragment {} is not valid media", path.display()); - None - } - }) - .collect(); - - if !result.is_empty() { - return result; + }) + .collect(); + + if !result.is_empty() { + return FragmentsInfo { + fragments: result, + init_segment, + }; + } } } - Self::probe_fragments_in_dir(dir) + FragmentsInfo { + fragments: Self::probe_fragments_in_dir(dir), + init_segment: None, + } } fn is_video_file(path: &Path) -> bool { path.extension() - .map(|e| e.eq_ignore_ascii_case("mp4")) + .map(|e| e.eq_ignore_ascii_case("mp4") || e.eq_ignore_ascii_case("m4s")) .unwrap_or(false) } @@ -301,7 +381,7 @@ impl RecoveryManager { .and_then(|e| e.to_str()) .map(|e| e.to_lowercase()); match ext.as_deref() { - Some("mp4") => match probe_video_can_decode(p) { + Some("mp4") | Some("m4s") => match probe_video_can_decode(p) { Ok(true) => true, Ok(false) => { debug!("Skipping {} - no decodable frames", p.display()); @@ -398,7 +478,7 @@ impl RecoveryManager { .join(format!("segment-{}", segment.index)); let display_output = segment_dir.join("display.mp4"); - if segment.display_fragments.len() == 1 { + if segment.display_fragments.len() == 1 && segment.display_init_segment.is_none() { let source = &segment.display_fragments[0]; if source != &display_output { info!("Moving single display fragment to {:?}", display_output); @@ -410,20 +490,39 @@ impl RecoveryManager { debug!("Failed to clean up display dir {:?}: {e}", display_dir); } } - } else if segment.display_fragments.len() > 1 { - info!( - "Concatenating {} display fragments to {:?}", - segment.display_fragments.len(), - display_output - ); - concatenate_video_fragments(&segment.display_fragments, &display_output) + } else if !segment.display_fragments.is_empty() { + if let Some(init_path) = &segment.display_init_segment { + info!( + "Concatenating {} M4S display segments with init to {:?}", + segment.display_fragments.len(), + display_output + ); + concatenate_m4s_segments_with_init( + init_path, + &segment.display_fragments, + &display_output, + ) .map_err(RecoveryError::VideoConcat)?; + } else { + info!( + "Concatenating {} display fragments to {:?}", + segment.display_fragments.len(), + display_output + ); + concatenate_video_fragments(&segment.display_fragments, &display_output) + .map_err(RecoveryError::VideoConcat)?; + } for fragment in &segment.display_fragments { if let Err(e) = std::fs::remove_file(fragment) { debug!("Failed to remove display fragment {:?}: {e}", fragment); } } + if let Some(init_path) = &segment.display_init_segment + && let Err(e) = std::fs::remove_file(init_path) + { + debug!("Failed to remove display init segment {:?}: {e}", init_path); + } let display_dir = segment_dir.join("display"); if display_dir.exists() && let Err(e) = std::fs::remove_dir_all(&display_dir) @@ -434,7 +533,7 @@ impl RecoveryManager { if let Some(camera_frags) = &segment.camera_fragments { let camera_output = segment_dir.join("camera.mp4"); - if camera_frags.len() == 1 { + if camera_frags.len() == 1 && segment.camera_init_segment.is_none() { let source = &camera_frags[0]; if source != &camera_output { info!("Moving single camera fragment to {:?}", camera_output); @@ -446,20 +545,35 @@ impl RecoveryManager { debug!("Failed to clean up camera dir {:?}: {e}", camera_dir); } } - } else if camera_frags.len() > 1 { - info!( - "Concatenating {} camera fragments to {:?}", - camera_frags.len(), - camera_output - ); - concatenate_video_fragments(camera_frags, &camera_output) - .map_err(RecoveryError::VideoConcat)?; + } else if !camera_frags.is_empty() { + if let Some(init_path) = &segment.camera_init_segment { + info!( + "Concatenating {} M4S camera segments with init to {:?}", + camera_frags.len(), + camera_output + ); + concatenate_m4s_segments_with_init(init_path, camera_frags, &camera_output) + .map_err(RecoveryError::VideoConcat)?; + } else { + info!( + "Concatenating {} camera fragments to {:?}", + camera_frags.len(), + camera_output + ); + concatenate_video_fragments(camera_frags, &camera_output) + .map_err(RecoveryError::VideoConcat)?; + } for fragment in camera_frags { if let Err(e) = std::fs::remove_file(fragment) { debug!("Failed to remove camera fragment {:?}: {e}", fragment); } } + if let Some(init_path) = &segment.camera_init_segment + && let Err(e) = std::fs::remove_file(init_path) + { + debug!("Failed to remove camera init segment {:?}: {e}", init_path); + } let camera_dir = segment_dir.join("camera"); if camera_dir.exists() && let Err(e) = std::fs::remove_dir_all(&camera_dir) @@ -652,6 +766,11 @@ impl RecoveryManager { fn build_recovered_meta( recording: &IncompleteRecording, ) -> Result { + let original_segments = match recording.meta.studio_meta() { + Some(StudioRecordingMeta::MultipleSegments { inner, .. }) => Some(&inner.segments), + _ => None, + }; + let segments: Vec = recording .recoverable_segments .iter() @@ -660,6 +779,9 @@ impl RecoveryManager { let segment_base = format!("content/segments/segment-{segment_index}"); let segment_dir = recording.project_path.join(&segment_base); + let original_segment = + original_segments.and_then(|segs| segs.get(segment_index as usize)); + let display_path = segment_dir.join("display.mp4"); let fps = get_video_fps(&display_path).unwrap_or(30); @@ -672,13 +794,18 @@ impl RecoveryManager { display: VideoMeta { path: RelativePathBuf::from(format!("{segment_base}/display.mp4")), fps, - start_time: None, + start_time: original_segment.and_then(|s| s.display.start_time), }, camera: if camera_path.exists() { Some(VideoMeta { path: RelativePathBuf::from(format!("{segment_base}/camera.mp4")), - fps: 30, - start_time: None, + fps: original_segment + .and_then(|s| s.camera.as_ref()) + .map(|c| c.fps) + .unwrap_or(30), + start_time: original_segment + .and_then(|s| s.camera.as_ref()) + .and_then(|c| c.start_time), }) } else { None @@ -686,7 +813,9 @@ impl RecoveryManager { mic: if mic_path.exists() { Some(AudioMeta { path: RelativePathBuf::from(format!("{segment_base}/audio-input.ogg")), - start_time: None, + start_time: original_segment + .and_then(|s| s.mic.as_ref()) + .and_then(|m| m.start_time), }) } else { None @@ -694,7 +823,9 @@ impl RecoveryManager { system_audio: if system_audio_path.exists() { Some(AudioMeta { path: RelativePathBuf::from(format!("{segment_base}/system_audio.ogg")), - start_time: None, + start_time: original_segment + .and_then(|s| s.system_audio.as_ref()) + .and_then(|a| a.start_time), }) } else { None diff --git a/crates/recording/src/sources/screen_capture/macos.rs b/crates/recording/src/sources/screen_capture/macos.rs index 51916b5e25..a5322af97e 100644 --- a/crates/recording/src/sources/screen_capture/macos.rs +++ b/crates/recording/src/sources/screen_capture/macos.rs @@ -13,7 +13,7 @@ use futures::{FutureExt as _, channel::mpsc, future::BoxFuture}; use std::{ sync::{ Arc, - atomic::{self, AtomicBool, AtomicU32}, + atomic::{self, AtomicBool, AtomicU32, AtomicU64}, }, time::Duration, }; @@ -24,6 +24,20 @@ use tokio_util::{ }; use tracing::{debug, warn}; +fn get_screen_buffer_size() -> usize { + std::env::var("CAP_SCREEN_BUFFER_SIZE") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(4) +} + +fn get_max_queue_depth() -> isize { + std::env::var("CAP_MAX_QUEUE_DEPTH") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(4) +} + #[derive(Debug)] pub struct CMSampleBufferCapture; @@ -31,7 +45,7 @@ impl ScreenCaptureFormat for CMSampleBufferCapture { type VideoFormat = cidre::arc::R; fn pixel_format() -> ffmpeg::format::Pixel { - ffmpeg::format::Pixel::BGRA + ffmpeg::format::Pixel::NV12 } fn audio_info() -> AudioInfo { @@ -68,8 +82,10 @@ impl ScreenCaptureConfig { &self, ) -> anyhow::Result<(VideoSourceConfig, Option)> { let (error_tx, error_rx) = broadcast::channel(1); - // Increased from 4 to 12 to provide more buffer tolerance for frame processing delays - let (video_tx, video_rx) = flume::bounded(12); + let buffer_size = get_screen_buffer_size(); + debug!(buffer_size = buffer_size, "Screen capture buffer size"); + let (video_tx, video_rx) = flume::bounded(buffer_size); + let drop_counter: Arc = Arc::new(AtomicU64::new(0)); let (mut audio_tx, audio_rx) = if self.system_audio { let (tx, rx) = mpsc::channel(32); (Some(tx), Some(rx)) @@ -129,8 +145,14 @@ impl ScreenCaptureConfig { debug!("size: {:?}", size); - let queue_depth = ((self.config.fps as f32 / 30.0 * 5.0).ceil() as isize).clamp(3, 8); - debug!("Using queue depth: {}", queue_depth); + let max_queue_depth = get_max_queue_depth(); + let queue_depth = + ((self.config.fps as f32 / 30.0 * 5.0).ceil() as isize).clamp(3, max_queue_depth); + debug!( + queue_depth = queue_depth, + max_queue_depth = max_queue_depth, + "Screen capture queue depth" + ); let mut settings = scap_screencapturekit::StreamCfgBuilder::default() .with_width(size.width() as usize) @@ -141,7 +163,7 @@ impl ScreenCaptureConfig { .with_queue_depth(queue_depth) .build(); - settings.set_pixel_format(cv::PixelFormat::_32_BGRA); + settings.set_pixel_format(cv::PixelFormat::_420V); settings.set_color_space_name(cg::color_space::names::srgb()); if let Some(crop_bounds) = self.config.crop_bounds { @@ -163,6 +185,7 @@ impl ScreenCaptureConfig { let builder = scap_screencapturekit::Capturer::builder(content_filter, settings) .with_output_sample_buf_cb({ let video_frame_count = video_frame_counter.clone(); + let drop_counter = drop_counter.clone(); move |frame| { let sample_buffer = frame.sample_buf(); @@ -185,10 +208,15 @@ impl ScreenCaptureConfig { video_frame_count.fetch_add(1, atomic::Ordering::Relaxed); - let _ = video_tx.try_send(VideoFrame { - sample_buf: sample_buffer.retained(), - timestamp, - }); + if video_tx + .try_send(VideoFrame { + sample_buf: sample_buffer.retained(), + timestamp, + }) + .is_err() + { + drop_counter.fetch_add(1, atomic::Ordering::Relaxed); + } } scap_screencapturekit::Frame::Audio(_) => { use ffmpeg::ChannelLayout; @@ -243,6 +271,7 @@ impl ScreenCaptureConfig { capturer: capturer.clone(), error_rx: error_rx.resubscribe(), video_frame_counter: video_frame_counter.clone(), + drop_counter, cancel_token: cancel_token.clone(), drop_guard: cancel_token.drop_guard(), }, @@ -341,12 +370,14 @@ pub struct VideoSourceConfig { cancel_token: CancellationToken, drop_guard: DropGuard, video_frame_counter: Arc, + drop_counter: Arc, } pub struct VideoSource { inner: ChannelVideoSource, capturer: Capturer, cancel_token: CancellationToken, video_frame_counter: Arc, + drop_counter: Arc, _drop_guard: DropGuard, } @@ -369,6 +400,7 @@ impl output_pipeline::VideoSource for VideoSource { cancel_token, drop_guard, video_frame_counter, + drop_counter, } = config; let monitor_capturer = capturer.clone(); @@ -415,6 +447,7 @@ impl output_pipeline::VideoSource for VideoSource { cancel_token, _drop_guard: drop_guard, video_frame_counter, + drop_counter, }) } @@ -424,13 +457,43 @@ impl output_pipeline::VideoSource for VideoSource { tokio::spawn({ let video_frame_count = self.video_frame_counter.clone(); + let drop_counter = self.drop_counter.clone(); async move { + let mut prev_frames = 0u32; + let mut prev_drops = 0u64; loop { tokio::time::sleep(Duration::from_secs(5)).await; - debug!( - "Captured {} frames", - video_frame_count.load(atomic::Ordering::Relaxed) - ); + let current_frames = video_frame_count.load(atomic::Ordering::Relaxed); + let current_drops = drop_counter.load(atomic::Ordering::Relaxed); + + let frame_delta = current_frames.saturating_sub(prev_frames); + let drop_delta = current_drops.saturating_sub(prev_drops); + + if frame_delta > 0 { + let drop_rate = 100.0 * drop_delta as f64 + / (frame_delta as f64 + drop_delta as f64); + if drop_rate > 5.0 { + warn!( + frames = frame_delta, + drops = drop_delta, + drop_rate_pct = format!("{:.1}%", drop_rate), + total_frames = current_frames, + total_drops = current_drops, + "Screen capture frame drop rate exceeds 5% threshold" + ); + } else { + debug!( + frames = frame_delta, + drops = drop_delta, + drop_rate_pct = format!("{:.1}%", drop_rate), + total_frames = current_frames, + "Screen capture stats" + ); + } + } + + prev_frames = current_frames; + prev_drops = current_drops; } } .with_cancellation_token_owned(self.cancel_token.clone()) diff --git a/crates/recording/src/studio_recording.rs b/crates/recording/src/studio_recording.rs index 325ee7509e..ae2f50a6ec 100644 --- a/crates/recording/src/studio_recording.rs +++ b/crates/recording/src/studio_recording.rs @@ -1,5 +1,5 @@ use crate::{ - ActorError, MediaError, RecordingBaseInputs, RecordingError, + ActorError, MediaError, RecordingBaseInputs, RecordingError, SharedPauseState, capture_pipeline::{ MakeCapturePipeline, ScreenCaptureMethod, Stop, target_to_display_and_crop, }, @@ -13,8 +13,8 @@ use crate::{ #[cfg(target_os = "macos")] use crate::output_pipeline::{ - AVFoundationCameraMuxer, AVFoundationCameraMuxerConfig, FragmentedAVFoundationCameraMuxer, - FragmentedAVFoundationCameraMuxerConfig, + AVFoundationCameraMuxer, AVFoundationCameraMuxerConfig, MacOSFragmentedM4SCameraMuxer, + MacOSFragmentedM4SCameraMuxerConfig, }; #[cfg(windows)] @@ -626,11 +626,8 @@ async fn stop_recording( let segment_metas: Vec<_> = futures::stream::iter(segments) .then(async |s| { - let to_start_time = |timestamp: Timestamp| { - timestamp - .duration_since(s.pipeline.start_time) - .as_secs_f64() - }; + let to_start_time = + |timestamp: Timestamp| timestamp.signed_duration_since_secs(s.pipeline.start_time); MultipleSegment { display: VideoMeta { @@ -843,11 +840,13 @@ async fn create_segment_pipeline( let (display, crop) = target_to_display_and_crop(&base_inputs.capture_target).context("target_display_crop")?; + let max_fps = if fragmented { 60 } else { 120 }; + let screen_config = ScreenCaptureConfig::::init( display, crop, !custom_cursor_capture, - 120, + max_fps, start_time.system_time(), base_inputs.capture_system_audio, #[cfg(windows)] @@ -868,11 +867,24 @@ async fn create_segment_pipeline( trace!("preparing segment pipeline {index}"); + #[cfg(target_os = "macos")] + let shared_pause_state = if fragmented { + Some(SharedPauseState::new(Arc::new( + std::sync::atomic::AtomicBool::new(false), + ))) + } else { + None + }; + + #[cfg(windows)] + let shared_pause_state: Option = None; + let screen = ScreenCaptureMethod::make_studio_mode_pipeline( capture_source, screen_output_path.clone(), start_time, fragmented, + shared_pause_state.clone(), #[cfg(windows)] encoder_preferences.clone(), ) @@ -887,9 +899,10 @@ async fn create_segment_pipeline( OutputPipeline::builder(fragments_dir) .with_video::(camera_feed) .with_timestamps(start_time) - .build::( - FragmentedAVFoundationCameraMuxerConfig::default(), - ) + .build::(MacOSFragmentedM4SCameraMuxerConfig { + shared_pause_state: shared_pause_state.clone(), + ..Default::default() + }) .instrument(error_span!("camera-out")) .await } else { @@ -940,7 +953,10 @@ async fn create_segment_pipeline( OutputPipeline::builder(fragments_dir) .with_audio_source::(mic_feed) .with_timestamps(start_time) - .build::(SegmentedAudioMuxerConfig::default()) + .build::(SegmentedAudioMuxerConfig { + shared_pause_state: shared_pause_state.clone(), + ..Default::default() + }) .instrument(error_span!("mic-out")) .await } else { @@ -962,7 +978,10 @@ async fn create_segment_pipeline( OutputPipeline::builder(fragments_dir) .with_audio_source::(system_audio_source) .with_timestamps(start_time) - .build::(SegmentedAudioMuxerConfig::default()) + .build::(SegmentedAudioMuxerConfig { + shared_pause_state: shared_pause_state.clone(), + ..Default::default() + }) .instrument(error_span!("system-audio-out")) .await } else { diff --git a/crates/rendering/src/decoder/avassetreader.rs b/crates/rendering/src/decoder/avassetreader.rs index 408490097b..abd8d6f804 100644 --- a/crates/rendering/src/decoder/avassetreader.rs +++ b/crates/rendering/src/decoder/avassetreader.rs @@ -18,50 +18,50 @@ use tokio::{runtime::Handle as TokioHandle, sync::oneshot}; use crate::{DecodedFrame, PixelFormat}; use super::frame_converter::{copy_bgra_to_rgba, copy_rgba_plane}; +use super::multi_position::{DecoderPoolManager, MultiPositionDecoderConfig, ScrubDetector}; use super::{DecoderInitResult, DecoderType, FRAME_CACHE_SIZE, VideoDecoderMessage, pts_to_frame}; +#[derive(Clone)] +struct FrameData { + data: Arc>, + y_stride: u32, + uv_stride: u32, +} + #[derive(Clone)] struct ProcessedFrame { _number: u32, - data: Arc>, width: u32, height: u32, format: PixelFormat, - y_stride: u32, - uv_stride: u32, - image_buf: Option>, + frame_data: FrameData, } impl ProcessedFrame { fn to_decoded_frame(&self) -> DecodedFrame { + let FrameData { + data, + y_stride, + uv_stride, + } = &self.frame_data; + match self.format { - PixelFormat::Rgba => DecodedFrame::new((*self.data).clone(), self.width, self.height), - PixelFormat::Nv12 => { - if let Some(image_buf) = &self.image_buf { - DecodedFrame::new_nv12_with_iosurface( - (*self.data).clone(), - self.width, - self.height, - self.y_stride, - self.uv_stride, - image_buf.retained(), - ) - } else { - DecodedFrame::new_nv12( - (*self.data).clone(), - self.width, - self.height, - self.y_stride, - self.uv_stride, - ) - } + PixelFormat::Rgba => { + DecodedFrame::new_with_arc(Arc::clone(data), self.width, self.height) } - PixelFormat::Yuv420p => DecodedFrame::new_yuv420p( - (*self.data).clone(), + PixelFormat::Nv12 => DecodedFrame::new_nv12_with_arc( + Arc::clone(data), + self.width, + self.height, + *y_stride, + *uv_stride, + ), + PixelFormat::Yuv420p => DecodedFrame::new_yuv420p_with_arc( + Arc::clone(data), self.width, self.height, - self.y_stride, - self.uv_stride, + *y_stride, + *uv_stride, ), } } @@ -202,28 +202,47 @@ impl ImageBufProcessor { } impl CachedFrame { - fn new(processor: &ImageBufProcessor, mut image_buf: R, number: u32) -> Self { + fn new(processor: &ImageBufProcessor, image_buf: R, number: u32) -> Self { let width = image_buf.width() as u32; let height = image_buf.height() as u32; - let (data, format, y_stride, uv_stride) = processor.extract_raw(&mut image_buf); - let retain_iosurface = format == PixelFormat::Nv12 && image_buf.io_surf().is_some(); + let pixel_format = + cap_video_decode::avassetreader::pixel_format_to_pixel(image_buf.pixel_format()); - let frame = ProcessedFrame { - _number: number, - data: Arc::new(data), - width, - height, - format, - y_stride, - uv_stride, - image_buf: if retain_iosurface { - Some(image_buf) - } else { - None - }, - }; - Self(frame) + match pixel_format { + format::Pixel::NV12 + | format::Pixel::RGBA + | format::Pixel::BGRA + | format::Pixel::YUV420P => { + let mut img = image_buf; + let (data, fmt, y_str, uv_str) = processor.extract_raw(&mut img); + Self(ProcessedFrame { + _number: number, + width, + height, + format: fmt, + frame_data: FrameData { + data: Arc::new(data), + y_stride: y_str, + uv_stride: uv_str, + }, + }) + } + _ => { + let black_frame = vec![0u8; (width * height * 4) as usize]; + Self(ProcessedFrame { + _number: number, + width, + height, + format: PixelFormat::Rgba, + frame_data: FrameData { + data: Arc::new(black_frame), + y_stride: width * 4, + uv_stride: 0, + }, + }) + } + } } fn data(&self) -> &ProcessedFrame { @@ -231,21 +250,147 @@ impl CachedFrame { } } -pub struct AVAssetReaderDecoder { +struct DecoderInstance { inner: cap_video_decode::AVAssetReaderDecoder, is_done: bool, + frames_iter_valid: bool, } -impl AVAssetReaderDecoder { - fn new(path: PathBuf, tokio_handle: TokioHandle) -> Result { +impl DecoderInstance { + fn new( + path: PathBuf, + tokio_handle: TokioHandle, + start_time: f32, + keyframe_index: Option, + ) -> Result { Ok(Self { - inner: cap_video_decode::AVAssetReaderDecoder::new(path, tokio_handle)?, + inner: cap_video_decode::AVAssetReaderDecoder::new_with_keyframe_index( + path, + tokio_handle, + start_time, + keyframe_index, + )?, is_done: false, + frames_iter_valid: true, }) } fn reset(&mut self, requested_time: f32) { - let _ = self.inner.reset(requested_time); + match self.inner.reset(requested_time) { + Ok(()) => { + self.is_done = false; + self.frames_iter_valid = true; + } + Err(e) => { + tracing::error!( + requested_time = requested_time, + error = %e, + "Failed to reset decoder, marking as invalid" + ); + self.is_done = true; + self.frames_iter_valid = false; + } + } + } + + fn current_position(&self) -> f32 { + self.inner.current_position_secs() + } +} + +pub struct AVAssetReaderDecoder { + decoders: Vec, + pool_manager: DecoderPoolManager, + active_decoder_idx: usize, + scrub_detector: ScrubDetector, +} + +impl AVAssetReaderDecoder { + fn new(path: PathBuf, tokio_handle: TokioHandle) -> Result { + let mut primary_decoder = + cap_video_decode::AVAssetReaderDecoder::new(path.clone(), tokio_handle.clone())?; + + let keyframe_index = primary_decoder.take_keyframe_index(); + let keyframe_index_arc: Option> = None; + + let fps = keyframe_index + .as_ref() + .map(|kf| kf.fps() as u32) + .unwrap_or(30); + let duration_secs = keyframe_index + .as_ref() + .map(|kf| kf.duration_secs()) + .unwrap_or(0.0); + + let config = MultiPositionDecoderConfig { + path: path.clone(), + tokio_handle: tokio_handle.clone(), + keyframe_index: keyframe_index_arc, + fps, + duration_secs, + }; + + let pool_manager = DecoderPoolManager::new(config); + + let primary_instance = DecoderInstance { + inner: primary_decoder, + is_done: false, + frames_iter_valid: true, + }; + + let mut decoders = vec![primary_instance]; + + let initial_positions = pool_manager.positions(); + for pos in initial_positions.iter().skip(1) { + let start_time = pos.position_secs; + match DecoderInstance::new(path.clone(), tokio_handle.clone(), start_time, None) { + Ok(instance) => { + decoders.push(instance); + tracing::info!( + position_secs = start_time, + decoder_index = decoders.len() - 1, + "Created additional decoder instance for multi-position pool" + ); + } + Err(e) => { + tracing::warn!( + position_secs = start_time, + error = %e, + "Failed to create additional decoder instance, continuing with fewer decoders" + ); + } + } + } + + tracing::info!( + decoder_count = decoders.len(), + fps = fps, + duration_secs = duration_secs, + "Initialized multi-position decoder pool" + ); + + Ok(Self { + decoders, + pool_manager, + active_decoder_idx: 0, + scrub_detector: ScrubDetector::new(), + }) + } + + fn select_best_decoder(&mut self, requested_time: f32) -> (usize, bool) { + let (best_id, _distance, needs_reset) = + self.pool_manager.find_best_decoder_for_time(requested_time); + + let decoder_idx = best_id.min(self.decoders.len().saturating_sub(1)); + + if needs_reset && decoder_idx < self.decoders.len() { + self.decoders[decoder_idx].reset(requested_time); + self.pool_manager + .update_decoder_position(best_id, self.decoders[decoder_idx].current_position()); + } + + self.active_decoder_idx = decoder_idx; + (decoder_idx, needs_reset) } pub fn spawn( @@ -276,8 +421,8 @@ impl AVAssetReaderDecoder { } }; - let video_width = this.inner.width(); - let video_height = this.inner.height(); + let video_width = this.decoders[0].inner.width(); + let video_height = this.decoders[0].inner.height(); let init_result = DecoderInitResult { width: video_width, @@ -291,193 +436,209 @@ impl AVAssetReaderDecoder { #[allow(unused)] let mut last_active_frame = None::; let last_sent_frame = Rc::new(RefCell::new(None::)); + let first_ever_frame = Rc::new(RefCell::new(None::)); - let mut frames = this.inner.frames(); let processor = ImageBufProcessor::new(); + struct PendingRequest { + frame: u32, + sender: oneshot::Sender, + } + while let Ok(r) = rx.recv() { + let mut pending_requests: Vec = Vec::with_capacity(8); + match r { VideoDecoderMessage::GetFrame(requested_time, sender) => { - if sender.is_closed() { - continue; + let frame = (requested_time * fps as f32).floor() as u32; + if !sender.is_closed() { + pending_requests.push(PendingRequest { frame, sender }); } + } + } - let requested_frame = (requested_time * fps as f32).floor() as u32; - - const BACKWARD_SEEK_TOLERANCE: u32 = 120; - let cache_frame_min_early = cache.keys().next().copied(); - let cache_frame_max_early = cache.keys().next_back().copied(); - - if let (Some(c_min), Some(_c_max)) = - (cache_frame_min_early, cache_frame_max_early) - { - let is_backward_within_tolerance = requested_frame < c_min - && requested_frame + BACKWARD_SEEK_TOLERANCE >= c_min; - if is_backward_within_tolerance - && let Some(closest_frame) = cache.get(&c_min) - { - let data = closest_frame.data().clone(); - if sender.send(data.to_decoded_frame()).is_err() { - debug!("frame receiver dropped before send"); - } - *last_sent_frame.borrow_mut() = Some(data); - continue; + while let Ok(msg) = rx.try_recv() { + match msg { + VideoDecoderMessage::GetFrame(requested_time, sender) => { + let frame = (requested_time * fps as f32).floor() as u32; + if !sender.is_closed() { + pending_requests.push(PendingRequest { frame, sender }); } } + } + } - let mut sender = if let Some(cached) = cache.get(&requested_frame) { - let data = cached.data().clone(); - if sender.send(data.to_decoded_frame()).is_err() { - debug!("frame receiver dropped before send"); - } - *last_sent_frame.borrow_mut() = Some(data); - continue; - } else { - let last_sent_frame = last_sent_frame.clone(); - Some(move |data: ProcessedFrame| { - *last_sent_frame.borrow_mut() = Some(data.clone()); - if sender.send(data.to_decoded_frame()).is_err() { - debug!("frame receiver dropped before send"); - } - }) - }; + pending_requests.sort_by_key(|r| r.frame); - let cache_min = requested_frame.saturating_sub(FRAME_CACHE_SIZE as u32 / 2); - let cache_max = requested_frame + FRAME_CACHE_SIZE as u32 / 2; - - let cache_frame_min = cache.keys().next().copied(); - let cache_frame_max = cache.keys().next_back().copied(); - - let needs_reset = - if let (Some(c_min), Some(c_max)) = (cache_frame_min, cache_frame_max) { - let is_backward_seek_beyond_tolerance = - requested_frame + BACKWARD_SEEK_TOLERANCE < c_min; - let is_forward_seek_beyond_cache = - requested_frame > c_max + FRAME_CACHE_SIZE as u32 / 4; - is_backward_seek_beyond_tolerance || is_forward_seek_beyond_cache - } else { - true - }; - - if needs_reset { - this.reset(requested_time); - frames = this.inner.frames(); - *last_sent_frame.borrow_mut() = None; - cache.retain(|&f, _| f >= cache_min && f <= cache_max); - } + let is_scrubbing = if let Some(first_req) = pending_requests.first() { + this.scrub_detector.record_request(first_req.frame) + } else { + false + }; + + let mut unfulfilled = Vec::with_capacity(pending_requests.len()); + let mut last_sent_data = None; + for request in pending_requests.drain(..) { + if let Some(cached) = cache.get(&request.frame) { + let data = cached.data().clone(); + let _ = request.sender.send(data.to_decoded_frame()); + last_sent_data = Some(data); + } else { + unfulfilled.push(request); + } + } + if let Some(data) = last_sent_data { + *last_sent_frame.borrow_mut() = Some(data); + } + pending_requests = unfulfilled; - last_active_frame = Some(requested_frame); + if pending_requests.is_empty() { + continue; + } - let mut exit = false; + let min_requested_frame = pending_requests.iter().map(|r| r.frame).min().unwrap(); + let max_requested_frame = pending_requests.iter().map(|r| r.frame).max().unwrap(); + let requested_frame = min_requested_frame; + let requested_time = requested_frame as f32 / fps as f32; - for frame in &mut frames { - let Ok(frame) = frame.map_err(|e| format!("read frame / {e}")) else { - continue; - }; + let (decoder_idx, was_reset) = this.select_best_decoder(requested_time); - let current_frame = pts_to_frame( - frame.pts().value, - Rational::new(1, frame.pts().scale), - fps, - ); + let cache_min = min_requested_frame.saturating_sub(FRAME_CACHE_SIZE as u32 / 2); + let cache_max = if is_scrubbing { + max_requested_frame + FRAME_CACHE_SIZE as u32 / 4 + } else { + max_requested_frame + FRAME_CACHE_SIZE as u32 / 2 + }; - let Some(frame) = frame.image_buf() else { - continue; - }; + if was_reset { + *last_sent_frame.borrow_mut() = None; + cache.retain(|&f, _| f >= cache_min && f <= cache_max); + } - let cache_frame = - CachedFrame::new(&processor, frame.retained(), current_frame); + last_active_frame = Some(requested_frame); + + let mut exit = false; + let mut frames_iterated = 0u32; + let mut last_decoded_position: Option = None; + + { + let decoder = &mut this.decoders[decoder_idx]; + let mut frames = decoder.inner.frames(); + + for frame in &mut frames { + let frame = match frame { + Ok(f) => f, + Err(e) => { + tracing::error!( + decoder_idx = decoder_idx, + frames_iterated = frames_iterated, + error = %e, + "Failed to read frame, skipping" + ); + continue; + } + }; + frames_iterated += 1; - this.is_done = false; + let current_frame = + pts_to_frame(frame.pts().value, Rational::new(1, frame.pts().scale), fps); - if let Some(most_recent_prev_frame) = - cache.iter().rev().find(|v| *v.0 < requested_frame) - && let Some(sender) = sender.take() - { - (sender)(most_recent_prev_frame.1.data().clone()); - } + let position_secs = current_frame as f32 / fps as f32; + last_decoded_position = Some(position_secs); - let exceeds_cache_bounds = current_frame > cache_max; - let too_small_for_cache_bounds = current_frame < cache_min; + let Some(frame) = frame.image_buf() else { + continue; + }; - if !too_small_for_cache_bounds { - if cache.len() >= FRAME_CACHE_SIZE { - if let Some(last_active_frame) = &last_active_frame { - let frame = if requested_frame > *last_active_frame { - *cache.keys().next().unwrap() - } else if requested_frame < *last_active_frame { - *cache.keys().next_back().unwrap() - } else { - let min = *cache.keys().min().unwrap(); - let max = *cache.keys().max().unwrap(); + let cache_frame = CachedFrame::new(&processor, frame.retained(), current_frame); - if current_frame > max { min } else { max } - }; + if first_ever_frame.borrow().is_none() { + *first_ever_frame.borrow_mut() = Some(cache_frame.data().clone()); + } - cache.remove(&frame); - } else { - cache.clear() - } - } + decoder.is_done = false; - cache.insert(current_frame, cache_frame.clone()); + let exceeds_cache_bounds = current_frame > cache_max; + let too_small_for_cache_bounds = current_frame < cache_min; - if current_frame == requested_frame - && let Some(sender) = sender.take() - { - (sender)(cache_frame.data().clone()); - break; + if !too_small_for_cache_bounds { + if cache.len() >= FRAME_CACHE_SIZE { + if let Some(last_active) = &last_active_frame { + let frame_to_remove = if requested_frame > *last_active { + *cache.keys().next().unwrap() + } else if requested_frame < *last_active { + *cache.keys().next_back().unwrap() + } else { + let min = *cache.keys().min().unwrap(); + let max = *cache.keys().max().unwrap(); + if current_frame > max { min } else { max } + }; + cache.remove(&frame_to_remove); + } else { + cache.clear() } } - if current_frame > requested_frame && sender.is_some() { - // not inlining this is important so that last_sent_frame is dropped before the sender is invoked - let last_sent_frame = last_sent_frame.borrow().clone(); - - if let Some((sender, last_sent_frame)) = - last_sent_frame.and_then(|l| Some((sender.take()?, l))) - { - // info!( - // "sending previous frame {} for {requested_frame}", - // last_sent_frame.0 - // ); - - (sender)(last_sent_frame); - } else if let Some(sender) = sender.take() { - (sender)(cache_frame.data().clone()); + cache.insert(current_frame, cache_frame.clone()); + + let mut remaining_requests = Vec::with_capacity(pending_requests.len()); + for req in pending_requests.drain(..) { + if req.frame == current_frame { + let data = cache_frame.data().clone(); + *last_sent_frame.borrow_mut() = Some(data.clone()); + let _ = req.sender.send(data.to_decoded_frame()); + } else if req.frame < current_frame { + if let Some(cached) = cache.get(&req.frame) { + let data = cached.data().clone(); + *last_sent_frame.borrow_mut() = Some(data.clone()); + let _ = req.sender.send(data.to_decoded_frame()); + } else if is_scrubbing { + let data = cache_frame.data().clone(); + *last_sent_frame.borrow_mut() = Some(data.clone()); + let _ = req.sender.send(data.to_decoded_frame()); + } + } else { + remaining_requests.push(req); } } + pending_requests = remaining_requests; + } - exit = exit || exceeds_cache_bounds; + *last_sent_frame.borrow_mut() = Some(cache_frame.data().clone()); - if exit { - break; - } + exit = exit || exceeds_cache_bounds; + + if is_scrubbing && frames_iterated > 3 { + break; } - this.is_done = true; - - let last_sent_frame = last_sent_frame.borrow().clone(); - if let Some(sender) = sender.take() { - if let Some(last_sent_frame) = last_sent_frame { - (sender)(last_sent_frame); - } else { - let black_frame_data = - vec![0u8; (video_width * video_height * 4) as usize]; - let black_frame = ProcessedFrame { - _number: requested_frame, - data: Arc::new(black_frame_data), - width: video_width, - height: video_height, - format: PixelFormat::Rgba, - y_stride: video_width * 4, - uv_stride: 0, - image_buf: None, - }; - (sender)(black_frame); - } + if pending_requests.is_empty() || exit { + break; } } + + decoder.is_done = true; + } + + if let Some(pos) = last_decoded_position { + this.pool_manager.update_decoder_position(decoder_idx, pos); + } + + for req in pending_requests.drain(..) { + if let Some(cached) = cache.get(&req.frame) { + let data = cached.data().clone(); + let _ = req.sender.send(data.to_decoded_frame()); + } else if let Some(last) = last_sent_frame.borrow().clone() { + if req.sender.send(last.to_decoded_frame()).is_err() {} + } else if let Some(first) = first_ever_frame.borrow().clone() { + if req.sender.send(first.to_decoded_frame()).is_err() {} + } else { + debug!( + decoder = _name, + requested_frame = req.frame, + "No frame available to send - request dropped" + ); + } } } } diff --git a/crates/rendering/src/decoder/ffmpeg.rs b/crates/rendering/src/decoder/ffmpeg.rs index 961e4cec28..b49ee06110 100644 --- a/crates/rendering/src/decoder/ffmpeg.rs +++ b/crates/rendering/src/decoder/ffmpeg.rs @@ -187,6 +187,7 @@ impl FfmpegDecoder { let mut last_active_frame = None::; let last_sent_frame = Rc::new(RefCell::new(None::)); + let first_ever_frame = Rc::new(RefCell::new(None::)); let mut frames = this.frames(); let mut converter = FrameConverter::new(); @@ -275,6 +276,14 @@ impl FfmpegDecoder { number: current_frame, }; + if first_ever_frame.borrow().is_none() { + let processed = cache_frame.process(&mut converter); + *first_ever_frame.borrow_mut() = Some(processed); + cache_frame = CachedFrame::Processed( + first_ever_frame.borrow().as_ref().unwrap().clone(), + ); + } + // Handles frame skips. // We use the cache instead of last_sent_frame as newer non-matching frames could have been decoded. if let Some(most_recent_prev_frame) = @@ -357,6 +366,11 @@ impl FfmpegDecoder { if let Some(sender) = sender.take() { if let Some(last_sent_frame) = last_sent_frame { (sender)(last_sent_frame); + } else if let Some(first_frame) = first_ever_frame.borrow().clone() { + debug!( + "Returning first decoded frame as fallback for request {requested_frame}" + ); + (sender)(first_frame); } else { debug!( "No frames available for request {requested_frame}, sending black frame" diff --git a/crates/rendering/src/decoder/mod.rs b/crates/rendering/src/decoder/mod.rs index bad0ddbaf1..81b92997a3 100644 --- a/crates/rendering/src/decoder/mod.rs +++ b/crates/rendering/src/decoder/mod.rs @@ -16,6 +16,8 @@ mod ffmpeg; mod frame_converter; #[cfg(target_os = "windows")] mod media_foundation; +#[cfg(target_os = "macos")] +pub mod multi_position; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum DecoderType { @@ -68,31 +70,9 @@ pub struct DecoderInitResult { pub decoder_type: DecoderType, } -#[cfg(target_os = "macos")] -use cidre::{arc::R, cv}; - #[cfg(target_os = "windows")] use windows::Win32::{Foundation::HANDLE, Graphics::Direct3D11::ID3D11Texture2D}; -#[cfg(target_os = "macos")] -pub struct SendableImageBuf(R); - -#[cfg(target_os = "macos")] -unsafe impl Send for SendableImageBuf {} -#[cfg(target_os = "macos")] -unsafe impl Sync for SendableImageBuf {} - -#[cfg(target_os = "macos")] -impl SendableImageBuf { - pub fn new(image_buf: R) -> Self { - Self(image_buf) - } - - pub fn inner(&self) -> &cv::ImageBuf { - &self.0 - } -} - #[cfg(target_os = "windows")] pub struct SendableD3D11Texture { texture: ID3D11Texture2D, @@ -172,8 +152,6 @@ pub struct DecodedFrame { format: PixelFormat, y_stride: u32, uv_stride: u32, - #[cfg(target_os = "macos")] - iosurface_backing: Option>, #[cfg(target_os = "windows")] d3d11_texture_backing: Option>, } @@ -200,8 +178,19 @@ impl DecodedFrame { format: PixelFormat::Rgba, y_stride: width * 4, uv_stride: 0, - #[cfg(target_os = "macos")] - iosurface_backing: None, + #[cfg(target_os = "windows")] + d3d11_texture_backing: None, + } + } + + pub fn new_with_arc(data: Arc>, width: u32, height: u32) -> Self { + Self { + data, + width, + height, + format: PixelFormat::Rgba, + y_stride: width * 4, + uv_stride: 0, #[cfg(target_os = "windows")] d3d11_texture_backing: None, } @@ -215,58 +204,66 @@ impl DecodedFrame { format: PixelFormat::Nv12, y_stride, uv_stride, - #[cfg(target_os = "macos")] - iosurface_backing: None, #[cfg(target_os = "windows")] d3d11_texture_backing: None, } } - pub fn new_yuv420p( - data: Vec, + pub fn new_nv12_with_arc( + data: Arc>, width: u32, height: u32, y_stride: u32, uv_stride: u32, ) -> Self { Self { - data: Arc::new(data), + data, width, height, - format: PixelFormat::Yuv420p, + format: PixelFormat::Nv12, y_stride, uv_stride, - #[cfg(target_os = "macos")] - iosurface_backing: None, #[cfg(target_os = "windows")] d3d11_texture_backing: None, } } - #[cfg(target_os = "macos")] - pub fn new_nv12_with_iosurface( + pub fn new_yuv420p( data: Vec, width: u32, height: u32, y_stride: u32, uv_stride: u32, - image_buf: R, ) -> Self { Self { data: Arc::new(data), width, height, - format: PixelFormat::Nv12, + format: PixelFormat::Yuv420p, y_stride, uv_stride, - iosurface_backing: Some(Arc::new(SendableImageBuf::new(image_buf))), + #[cfg(target_os = "windows")] + d3d11_texture_backing: None, } } - #[cfg(target_os = "macos")] - #[allow(clippy::redundant_closure)] - pub fn iosurface_backing(&self) -> Option<&cv::ImageBuf> { - self.iosurface_backing.as_ref().map(|b| b.inner()) + pub fn new_yuv420p_with_arc( + data: Arc>, + width: u32, + height: u32, + y_stride: u32, + uv_stride: u32, + ) -> Self { + Self { + data, + width, + height, + format: PixelFormat::Yuv420p, + y_stride, + uv_stride, + #[cfg(target_os = "windows")] + d3d11_texture_backing: None, + } } #[cfg(target_os = "windows")] @@ -451,7 +448,7 @@ pub fn pts_to_frame(pts: i64, time_base: Rational, fps: u32) -> u32 { .round() as u32 } -pub const FRAME_CACHE_SIZE: usize = 750; +pub const FRAME_CACHE_SIZE: usize = 150; #[derive(Clone)] pub struct AsyncVideoDecoderHandle { @@ -474,7 +471,16 @@ impl AsyncVideoDecoderHandle { return None; } - rx.await.ok() + match tokio::time::timeout(std::time::Duration::from_millis(500), rx).await { + Ok(result) => result.ok(), + Err(_) => { + debug!( + adjusted_time = adjusted_time, + "get_frame timed out after 500ms" + ); + None + } + } } pub fn get_time(&self, time: f32) -> f32 { diff --git a/crates/rendering/src/decoder/multi_position.rs b/crates/rendering/src/decoder/multi_position.rs new file mode 100644 index 0000000000..4f8e294a2f --- /dev/null +++ b/crates/rendering/src/decoder/multi_position.rs @@ -0,0 +1,238 @@ +use std::{collections::BTreeMap, path::PathBuf, sync::Arc}; + +use tokio::runtime::Handle as TokioHandle; + +use cap_video_decode::avassetreader::KeyframeIndex; + +pub const MAX_DECODER_POOL_SIZE: usize = 3; +pub const REPOSITION_THRESHOLD_SECS: f32 = 5.0; + +pub struct DecoderPosition { + pub id: usize, + pub position_secs: f32, + pub last_access_time: std::time::Instant, + pub access_count: u64, +} + +impl DecoderPosition { + pub fn new(id: usize, position_secs: f32) -> Self { + Self { + id, + position_secs, + last_access_time: std::time::Instant::now(), + access_count: 0, + } + } + + pub fn touch(&mut self) { + self.last_access_time = std::time::Instant::now(); + self.access_count += 1; + } +} + +pub struct MultiPositionDecoderConfig { + pub path: PathBuf, + pub tokio_handle: TokioHandle, + pub keyframe_index: Option>, + pub fps: u32, + pub duration_secs: f64, +} + +pub struct DecoderPoolManager { + config: MultiPositionDecoderConfig, + positions: Vec, + access_history: BTreeMap, + total_accesses: u64, +} + +impl DecoderPoolManager { + pub fn new(config: MultiPositionDecoderConfig) -> Self { + let initial_positions = Self::calculate_initial_positions(&config); + + let positions: Vec = initial_positions + .into_iter() + .enumerate() + .map(|(id, pos)| DecoderPosition::new(id, pos)) + .collect(); + + Self { + config, + positions, + access_history: BTreeMap::new(), + total_accesses: 0, + } + } + + fn calculate_initial_positions(config: &MultiPositionDecoderConfig) -> Vec { + if let Some(ref kf_index) = config.keyframe_index { + let strategic = kf_index.get_strategic_positions(MAX_DECODER_POOL_SIZE); + strategic.into_iter().map(|t| t as f32).collect() + } else { + let duration = config.duration_secs as f32; + if duration <= 0.0 { + vec![0.0] + } else { + (0..MAX_DECODER_POOL_SIZE) + .map(|i| { + let frac = i as f32 / MAX_DECODER_POOL_SIZE as f32; + (duration * frac).min(duration) + }) + .collect() + } + } + } + + pub fn find_best_decoder_for_time(&mut self, requested_time: f32) -> (usize, f32, bool) { + self.total_accesses += 1; + + let frame = (requested_time * self.config.fps as f32).floor() as u32; + *self.access_history.entry(frame).or_insert(0) += 1; + + let mut best_decoder_id = 0; + let mut best_distance = f32::MAX; + let mut needs_reset = true; + + for position in &self.positions { + let distance = (position.position_secs - requested_time).abs(); + let is_usable = position.position_secs <= requested_time + && (requested_time - position.position_secs) < REPOSITION_THRESHOLD_SECS; + + if is_usable && distance < best_distance { + best_distance = distance; + best_decoder_id = position.id; + needs_reset = false; + } + } + + if needs_reset { + for position in &self.positions { + let distance = (position.position_secs - requested_time).abs(); + if distance < best_distance { + best_distance = distance; + best_decoder_id = position.id; + } + } + } + + if let Some(pos) = self.positions.iter_mut().find(|p| p.id == best_decoder_id) { + pos.touch(); + } + + (best_decoder_id, best_distance, needs_reset) + } + + pub fn update_decoder_position(&mut self, decoder_id: usize, new_position: f32) { + if let Some(pos) = self.positions.iter_mut().find(|p| p.id == decoder_id) { + pos.position_secs = new_position; + } + } + + pub fn should_rebalance(&self) -> bool { + self.total_accesses > 0 && self.total_accesses.is_multiple_of(100) + } + + pub fn get_rebalance_positions(&self) -> Vec { + if self.access_history.is_empty() { + return self.positions.iter().map(|p| p.position_secs).collect(); + } + + let mut hotspots: Vec<(u32, u64)> = self + .access_history + .iter() + .map(|(&frame, &count)| (frame, count)) + .collect(); + hotspots.sort_by(|a, b| b.1.cmp(&a.1)); + + let top_hotspots: Vec = hotspots + .into_iter() + .take(MAX_DECODER_POOL_SIZE) + .map(|(frame, _)| frame as f32 / self.config.fps as f32) + .collect(); + + if top_hotspots.len() < MAX_DECODER_POOL_SIZE { + let mut result = top_hotspots; + let remaining = MAX_DECODER_POOL_SIZE - result.len(); + let duration = self.config.duration_secs as f32; + for i in 0..remaining { + let frac = (i + 1) as f32 / (remaining + 1) as f32; + result.push(duration * frac); + } + result + } else { + top_hotspots + } + } + + pub fn positions(&self) -> &[DecoderPosition] { + &self.positions + } + + pub fn config(&self) -> &MultiPositionDecoderConfig { + &self.config + } +} + +pub struct ScrubDetector { + last_request_time: std::time::Instant, + last_frame: u32, + request_rate: f64, + is_scrubbing: bool, + scrub_start_time: Option, +} + +impl ScrubDetector { + const SCRUB_THRESHOLD_RATE: f64 = 5.0; + const SCRUB_COOLDOWN_MS: u64 = 150; + + pub fn new() -> Self { + Self { + last_request_time: std::time::Instant::now(), + last_frame: 0, + request_rate: 0.0, + is_scrubbing: false, + scrub_start_time: None, + } + } + + pub fn record_request(&mut self, frame: u32) -> bool { + let now = std::time::Instant::now(); + let elapsed = now.duration_since(self.last_request_time); + let elapsed_secs = elapsed.as_secs_f64().max(0.001); + + let frame_delta = frame.abs_diff(self.last_frame); + + let instantaneous_rate = frame_delta as f64 / elapsed_secs; + self.request_rate = self.request_rate * 0.7 + instantaneous_rate * 0.3; + + let _was_scrubbing = self.is_scrubbing; + + if self.request_rate > Self::SCRUB_THRESHOLD_RATE && frame_delta > 1 { + self.is_scrubbing = true; + if self.scrub_start_time.is_none() { + self.scrub_start_time = Some(now); + } + } else if elapsed.as_millis() as u64 > Self::SCRUB_COOLDOWN_MS { + self.is_scrubbing = false; + self.scrub_start_time = None; + } + + self.last_request_time = now; + self.last_frame = frame; + + self.is_scrubbing + } + + pub fn is_scrubbing(&self) -> bool { + self.is_scrubbing + } + + pub fn scrub_duration(&self) -> Option { + self.scrub_start_time.map(|start| start.elapsed()) + } +} + +impl Default for ScrubDetector { + fn default() -> Self { + Self::new() + } +} diff --git a/crates/rendering/src/layers/camera.rs b/crates/rendering/src/layers/camera.rs index 17754734e0..a0c1d85c3c 100644 --- a/crates/rendering/src/layers/camera.rs +++ b/crates/rendering/src/layers/camera.rs @@ -14,7 +14,7 @@ pub struct CameraLayer { bind_groups: [Option; 2], pipeline: CompositeVideoFramePipeline, hidden: bool, - last_frame_ptr: usize, + last_recording_time: Option, yuv_converter: YuvToRgbaConverter, } @@ -50,7 +50,7 @@ impl CameraLayer { bind_groups: [bind_group_0, bind_group_1], pipeline, hidden: false, - last_frame_ptr: 0, + last_recording_time: None, yuv_converter, } } @@ -59,19 +59,31 @@ impl CameraLayer { &mut self, device: &wgpu::Device, queue: &wgpu::Queue, - data: Option<(CompositeVideoFrameUniforms, XY, &DecodedFrame)>, + uniforms: Option, + frame_data: Option<(XY, &DecodedFrame, f32)>, ) { - self.hidden = data.is_none(); + let Some(uniforms) = uniforms else { + self.hidden = true; + return; + }; + + let has_previous_frame = self.last_recording_time.is_some(); + self.hidden = frame_data.is_none() && !has_previous_frame; + + queue.write_buffer(&self.uniforms_buffer, 0, bytemuck::cast_slice(&[uniforms])); - let Some((uniforms, frame_size, camera_frame)) = data else { + let Some((frame_size, camera_frame, recording_time)) = frame_data else { return; }; - let frame_data = camera_frame.data(); - let frame_ptr = frame_data.as_ptr() as usize; + let frame_data_bytes = camera_frame.data(); let format = camera_frame.format(); - if frame_ptr != self.last_frame_ptr { + let is_same_frame = self + .last_recording_time + .is_some_and(|last| (last - recording_time).abs() < 0.001); + + if !is_same_frame { let next_texture = 1 - self.current_texture; if self.frame_textures[next_texture].width() != frame_size.x @@ -104,7 +116,7 @@ impl CameraLayer { origin: wgpu::Origin3d::ZERO, aspect: wgpu::TextureAspect::All, }, - frame_data, + frame_data_bytes, wgpu::TexelCopyBufferLayout { offset: 0, bytes_per_row: Some(src_bytes_per_row), @@ -162,11 +174,9 @@ impl CameraLayer { } } - self.last_frame_ptr = frame_ptr; + self.last_recording_time = Some(recording_time); self.current_texture = next_texture; } - - queue.write_buffer(&self.uniforms_buffer, 0, bytemuck::cast_slice(&[uniforms])); } fn copy_from_yuv_output( diff --git a/crates/rendering/src/layers/display.rs b/crates/rendering/src/layers/display.rs index 78e3dd9fce..c799f8dfad 100644 --- a/crates/rendering/src/layers/display.rs +++ b/crates/rendering/src/layers/display.rs @@ -80,18 +80,9 @@ impl DisplayLayer { let format = segment_frames.screen_frame.format(); let current_recording_time = segment_frames.recording_time; - tracing::trace!( - format = ?format, - actual_width, - actual_height, - frame_data_len = frame_data.len(), - recording_time = current_recording_time, - "DisplayLayer::prepare - frame info" - ); - let skipped = self .last_recording_time - .is_some_and(|last| (last - current_recording_time).abs() < f32::EPSILON); + .is_some_and(|last| (last - current_recording_time).abs() < 0.001); if !skipped { let next_texture = 1 - self.current_texture; @@ -143,26 +134,9 @@ impl DisplayLayer { PixelFormat::Nv12 => { let screen_frame = &segment_frames.screen_frame; - #[cfg(target_os = "macos")] - let iosurface_result = screen_frame.iosurface_backing().map(|image_buf| { - self.yuv_converter - .convert_nv12_from_iosurface(device, queue, image_buf) - }); - #[cfg(target_os = "macos")] if !self.prefer_cpu_conversion { - if let Some(Ok(_)) = iosurface_result { - if self.yuv_converter.output_texture().is_some() { - self.pending_copy = Some(PendingTextureCopy { - width: frame_size.x, - height: frame_size.y, - dst_texture_index: next_texture, - }); - true - } else { - false - } - } else if let (Some(y_data), Some(uv_data)) = + if let (Some(y_data), Some(uv_data)) = (screen_frame.y_plane(), screen_frame.uv_plane()) { let y_stride = screen_frame.y_stride(); @@ -492,7 +466,6 @@ impl DisplayLayer { pub fn copy_to_texture(&mut self, encoder: &mut wgpu::CommandEncoder) { let Some(pending) = self.pending_copy.take() else { - tracing::trace!("copy_to_texture: no pending copy"); return; }; @@ -524,10 +497,6 @@ impl DisplayLayer { pub fn render(&self, pass: &mut wgpu::RenderPass<'_>) { if let Some(bind_group) = &self.bind_groups[self.current_texture] { - tracing::trace!( - current_texture_index = self.current_texture, - "DisplayLayer::render - rendering with bind group" - ); pass.set_pipeline(&self.pipeline.render_pipeline); pass.set_bind_group(0, bind_group, &[]); pass.draw(0..3, 0..1); diff --git a/crates/rendering/src/lib.rs b/crates/rendering/src/lib.rs index 4da4bc794f..48f2b20889 100644 --- a/crates/rendering/src/lib.rs +++ b/crates/rendering/src/lib.rs @@ -158,7 +158,7 @@ impl RecordingSegmentDecoders { segment.camera.as_ref().unwrap().fps } StudioRecordingMeta::MultipleSegments { inner, .. } => { - inner.segments[0].camera.as_ref().unwrap().fps + inner.segments[segment_i].camera.as_ref().unwrap().fps } }, match &meta { @@ -1694,25 +1694,25 @@ impl RendererLayers { self.camera.prepare( &constants.device, &constants.queue, - (|| { - Some(( - uniforms.camera?, - constants.options.camera_size?, - segment_frames.camera_frame.as_ref()?, - )) - })(), + uniforms.camera, + constants.options.camera_size.and_then(|size| { + segment_frames + .camera_frame + .as_ref() + .map(|frame| (size, frame, segment_frames.recording_time)) + }), ); self.camera_only.prepare( &constants.device, &constants.queue, - (|| { - Some(( - uniforms.camera_only?, - constants.options.camera_size?, - segment_frames.camera_frame.as_ref()?, - )) - })(), + uniforms.camera_only, + constants.options.camera_size.and_then(|size| { + segment_frames + .camera_frame + .as_ref() + .map(|frame| (size, frame, segment_frames.recording_time)) + }), ); self.text.prepare( @@ -1780,12 +1780,6 @@ impl RendererLayers { } let should_render = uniforms.scene.should_render_screen(); - tracing::trace!( - should_render_screen = should_render, - screen_opacity = uniforms.scene.screen_opacity, - screen_blur = uniforms.scene.screen_blur, - "RendererLayers::render - checking should_render_screen" - ); if should_render { let mut pass = render_pass!(session.current_texture_view(), wgpu::LoadOp::Load); diff --git a/crates/rendering/src/yuv_converter.rs b/crates/rendering/src/yuv_converter.rs index 76126c7cc9..a4fe8e58ec 100644 --- a/crates/rendering/src/yuv_converter.rs +++ b/crates/rendering/src/yuv_converter.rs @@ -133,6 +133,122 @@ fn validate_dimensions( Ok((new_width, new_height, true)) } +struct BindGroupCache { + nv12_bind_groups: [Option; 2], + yuv420p_bind_groups: [Option; 2], + cached_width: u32, + cached_height: u32, +} + +impl BindGroupCache { + fn new() -> Self { + Self { + nv12_bind_groups: [None, None], + yuv420p_bind_groups: [None, None], + cached_width: 0, + cached_height: 0, + } + } + + fn invalidate(&mut self) { + self.nv12_bind_groups = [None, None]; + self.yuv420p_bind_groups = [None, None]; + self.cached_width = 0; + self.cached_height = 0; + } + + #[allow(clippy::too_many_arguments)] + fn get_or_create_nv12( + &mut self, + device: &wgpu::Device, + layout: &wgpu::BindGroupLayout, + y_view: &wgpu::TextureView, + uv_view: &wgpu::TextureView, + output_view: &wgpu::TextureView, + output_index: usize, + width: u32, + height: u32, + ) -> &wgpu::BindGroup { + if self.cached_width != width || self.cached_height != height { + self.invalidate(); + self.cached_width = width; + self.cached_height = height; + } + + if self.nv12_bind_groups[output_index].is_none() { + self.nv12_bind_groups[output_index] = + Some(device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("NV12 Converter Bind Group (Cached)"), + layout, + entries: &[ + wgpu::BindGroupEntry { + binding: 0, + resource: wgpu::BindingResource::TextureView(y_view), + }, + wgpu::BindGroupEntry { + binding: 1, + resource: wgpu::BindingResource::TextureView(uv_view), + }, + wgpu::BindGroupEntry { + binding: 2, + resource: wgpu::BindingResource::TextureView(output_view), + }, + ], + })); + } + + self.nv12_bind_groups[output_index].as_ref().unwrap() + } + + #[allow(clippy::too_many_arguments)] + fn get_or_create_yuv420p( + &mut self, + device: &wgpu::Device, + layout: &wgpu::BindGroupLayout, + y_view: &wgpu::TextureView, + u_view: &wgpu::TextureView, + v_view: &wgpu::TextureView, + output_view: &wgpu::TextureView, + output_index: usize, + width: u32, + height: u32, + ) -> &wgpu::BindGroup { + if self.cached_width != width || self.cached_height != height { + self.invalidate(); + self.cached_width = width; + self.cached_height = height; + } + + if self.yuv420p_bind_groups[output_index].is_none() { + self.yuv420p_bind_groups[output_index] = + Some(device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("YUV420P Converter Bind Group (Cached)"), + layout, + entries: &[ + wgpu::BindGroupEntry { + binding: 0, + resource: wgpu::BindingResource::TextureView(y_view), + }, + wgpu::BindGroupEntry { + binding: 1, + resource: wgpu::BindingResource::TextureView(u_view), + }, + wgpu::BindGroupEntry { + binding: 2, + resource: wgpu::BindingResource::TextureView(v_view), + }, + wgpu::BindGroupEntry { + binding: 3, + resource: wgpu::BindingResource::TextureView(output_view), + }, + ], + })); + } + + self.yuv420p_bind_groups[output_index].as_ref().unwrap() + } +} + pub struct YuvToRgbaConverter { nv12_pipeline: wgpu::ComputePipeline, yuv420p_pipeline: wgpu::ComputePipeline, @@ -152,6 +268,7 @@ pub struct YuvToRgbaConverter { allocated_width: u32, allocated_height: u32, gpu_max_texture_size: u32, + bind_group_cache: BindGroupCache, #[cfg(target_os = "macos")] iosurface_cache: Option, #[cfg(target_os = "windows")] @@ -331,6 +448,7 @@ impl YuvToRgbaConverter { allocated_width: initial_width, allocated_height: initial_height, gpu_max_texture_size, + bind_group_cache: BindGroupCache::new(), #[cfg(target_os = "macos")] iosurface_cache: IOSurfaceTextureCache::new(), #[cfg(target_os = "windows")] @@ -507,6 +625,7 @@ impl YuvToRgbaConverter { self.output_views = output_views; self.allocated_width = new_width; self.allocated_height = new_height; + self.bind_group_cache.invalidate(); } pub fn gpu_max_texture_size(&self) -> u32 { @@ -574,24 +693,17 @@ impl YuvToRgbaConverter { }, ); - let bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor { - label: Some("NV12 Converter Bind Group"), - layout: &self.nv12_bind_group_layout, - entries: &[ - wgpu::BindGroupEntry { - binding: 0, - resource: wgpu::BindingResource::TextureView(&self.y_view), - }, - wgpu::BindGroupEntry { - binding: 1, - resource: wgpu::BindingResource::TextureView(&self.uv_view), - }, - wgpu::BindGroupEntry { - binding: 2, - resource: wgpu::BindingResource::TextureView(self.current_output_view()), - }, - ], - }); + let output_index = self.current_output; + let bind_group = self.bind_group_cache.get_or_create_nv12( + device, + &self.nv12_bind_group_layout, + &self.y_view, + &self.uv_view, + &self.output_views[output_index], + output_index, + self.allocated_width, + self.allocated_height, + ); let mut encoder = device.create_command_encoder(&wgpu::CommandEncoderDescriptor { label: Some("NV12 Conversion Encoder"), @@ -603,7 +715,7 @@ impl YuvToRgbaConverter { ..Default::default() }); compute_pass.set_pipeline(&self.nv12_pipeline); - compute_pass.set_bind_group(0, &bind_group, &[]); + compute_pass.set_bind_group(0, bind_group, &[]); compute_pass.dispatch_workgroups(width.div_ceil(8), height.div_ceil(8), 1); } @@ -740,28 +852,18 @@ impl YuvToRgbaConverter { "V", )?; - let bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor { - label: Some("YUV420P Converter Bind Group"), - layout: &self.yuv420p_bind_group_layout, - entries: &[ - wgpu::BindGroupEntry { - binding: 0, - resource: wgpu::BindingResource::TextureView(&self.y_view), - }, - wgpu::BindGroupEntry { - binding: 1, - resource: wgpu::BindingResource::TextureView(&self.u_view), - }, - wgpu::BindGroupEntry { - binding: 2, - resource: wgpu::BindingResource::TextureView(&self.v_view), - }, - wgpu::BindGroupEntry { - binding: 3, - resource: wgpu::BindingResource::TextureView(self.current_output_view()), - }, - ], - }); + let output_index = self.current_output; + let bind_group = self.bind_group_cache.get_or_create_yuv420p( + device, + &self.yuv420p_bind_group_layout, + &self.y_view, + &self.u_view, + &self.v_view, + &self.output_views[output_index], + output_index, + self.allocated_width, + self.allocated_height, + ); let mut encoder = device.create_command_encoder(&wgpu::CommandEncoderDescriptor { label: Some("YUV420P Conversion Encoder"), @@ -773,7 +875,7 @@ impl YuvToRgbaConverter { ..Default::default() }); compute_pass.set_pipeline(&self.yuv420p_pipeline); - compute_pass.set_bind_group(0, &bind_group, &[]); + compute_pass.set_bind_group(0, bind_group, &[]); compute_pass.dispatch_workgroups(width.div_ceil(8), height.div_ceil(8), 1); } diff --git a/crates/timestamp/src/lib.rs b/crates/timestamp/src/lib.rs index 47b4b19013..4a37b8295c 100644 --- a/crates/timestamp/src/lib.rs +++ b/crates/timestamp/src/lib.rs @@ -45,6 +45,31 @@ impl Timestamp { } } + pub fn signed_duration_since_secs(&self, start: Timestamps) -> f64 { + match self { + Self::Instant(instant) => { + if let Some(duration) = instant.checked_duration_since(start.instant) { + duration.as_secs_f64() + } else { + let reverse = start.instant.duration_since(*instant); + -(reverse.as_secs_f64()) + } + } + Self::SystemTime(time) => match time.duration_since(start.system_time) { + Ok(duration) => duration.as_secs_f64(), + Err(e) => -(e.duration().as_secs_f64()), + }, + #[cfg(windows)] + Self::PerformanceCounter(counter) => { + counter.signed_duration_since_secs(start.performance_counter) + } + #[cfg(target_os = "macos")] + Self::MachAbsoluteTime(time) => { + time.signed_duration_since_secs(start.mach_absolute_time) + } + } + } + pub fn from_cpal(instant: cpal::StreamInstant) -> Self { #[cfg(windows)] { diff --git a/crates/timestamp/src/macos.rs b/crates/timestamp/src/macos.rs index b726533501..0a2e0539d2 100644 --- a/crates/timestamp/src/macos.rs +++ b/crates/timestamp/src/macos.rs @@ -39,6 +39,19 @@ impl MachAbsoluteTimestamp { Some(Duration::from_nanos((diff as f64 * freq) as u64)) } + pub fn signed_duration_since_secs(&self, other: Self) -> f64 { + let info = TimeBaseInfo::new(); + let freq = info.numer as f64 / info.denom as f64; + + let nanos = if self.0 >= other.0 { + ((self.0 - other.0) as f64 * freq) as i64 + } else { + -(((other.0 - self.0) as f64 * freq) as i64) + }; + + nanos as f64 / 1_000_000_000.0 + } + pub fn from_cpal(instant: cpal::StreamInstant) -> Self { use cpal::host::coreaudio::StreamInstantExt; diff --git a/crates/timestamp/src/win.rs b/crates/timestamp/src/win.rs index d3efe2d444..8a111489bd 100644 --- a/crates/timestamp/src/win.rs +++ b/crates/timestamp/src/win.rs @@ -65,6 +65,12 @@ impl PerformanceCounterTimestamp { } } + pub fn signed_duration_since_secs(&self, other: Self) -> f64 { + let freq = perf_freq() as f64; + let diff = self.0 as f64 - other.0 as f64; + diff / freq + } + pub fn now() -> Self { let mut value = 0; unsafe { QueryPerformanceCounter(&mut value).unwrap() }; diff --git a/crates/video-decode/src/avassetreader.rs b/crates/video-decode/src/avassetreader.rs index 8471701bb9..189d60e011 100644 --- a/crates/video-decode/src/avassetreader.rs +++ b/crates/video-decode/src/avassetreader.rs @@ -1,4 +1,4 @@ -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use cidre::{ arc::{self, R}, @@ -10,6 +10,172 @@ use cidre::{ use ffmpeg::{codec as avcodec, format as avformat}; use tokio::runtime::Handle as TokioHandle; +pub struct KeyframeIndex { + keyframes: Vec<(u32, f64)>, + fps: f64, + duration_secs: f64, +} + +impl KeyframeIndex { + pub fn build(path: &Path) -> Result { + let build_start = std::time::Instant::now(); + + let input = avformat::input(path) + .map_err(|e| format!("Failed to open video for keyframe scan: {e}"))?; + + let video_stream = input + .streams() + .best(ffmpeg::media::Type::Video) + .ok_or("No video stream found")?; + + let stream_index = video_stream.index(); + let time_base = video_stream.time_base(); + let fps = { + let rate = video_stream.avg_frame_rate(); + if rate.denominator() == 0 { + 30.0 + } else { + rate.numerator() as f64 / rate.denominator() as f64 + } + }; + + let duration_secs = { + let duration = video_stream.duration(); + if duration > 0 { + duration as f64 * time_base.numerator() as f64 / time_base.denominator() as f64 + } else { + 0.0 + } + }; + + let mut keyframes = Vec::new(); + + let mut input = + avformat::input(path).map_err(|e| format!("Failed to reopen video for scan: {e}"))?; + + for (stream, packet) in input.packets() { + if stream.index() != stream_index { + continue; + } + + if packet.is_key() { + let pts = packet.pts().unwrap_or(0); + let time_secs = + pts as f64 * time_base.numerator() as f64 / time_base.denominator() as f64; + let frame_number = (time_secs * fps).round() as u32; + keyframes.push((frame_number, time_secs)); + } + } + + let elapsed = build_start.elapsed(); + tracing::info!( + path = %path.display(), + keyframe_count = keyframes.len(), + fps = fps, + duration_secs = duration_secs, + build_ms = elapsed.as_millis(), + "Built keyframe index" + ); + + Ok(Self { + keyframes, + fps, + duration_secs, + }) + } + + pub fn nearest_keyframe_before(&self, target_frame: u32) -> Option<(u32, f64)> { + if self.keyframes.is_empty() { + return None; + } + + let pos = self + .keyframes + .binary_search_by_key(&target_frame, |(frame, _)| *frame); + + match pos { + Ok(i) => Some(self.keyframes[i]), + Err(0) => None, + Err(i) => Some(self.keyframes[i - 1]), + } + } + + pub fn nearest_keyframe_after(&self, target_frame: u32) -> Option<(u32, f64)> { + if self.keyframes.is_empty() { + return None; + } + + let pos = self + .keyframes + .binary_search_by_key(&target_frame, |(frame, _)| *frame); + + let idx = match pos { + Ok(i) => { + if i + 1 < self.keyframes.len() { + i + 1 + } else { + i + } + } + Err(i) => { + if i < self.keyframes.len() { + i + } else { + return None; + } + } + }; + + Some(self.keyframes[idx]) + } + + pub fn get_strategic_positions(&self, num_positions: usize) -> Vec { + if self.keyframes.is_empty() || num_positions == 0 { + return vec![0.0]; + } + + let total_keyframes = self.keyframes.len(); + if total_keyframes <= num_positions { + return self.keyframes.iter().map(|(_, time)| *time).collect(); + } + + let step = total_keyframes / num_positions; + self.keyframes + .iter() + .step_by(step.max(1)) + .take(num_positions) + .map(|(_, time)| *time) + .collect() + } + + pub fn fps(&self) -> f64 { + self.fps + } + + pub fn duration_secs(&self) -> f64 { + self.duration_secs + } + + pub fn keyframe_count(&self) -> usize { + self.keyframes.len() + } + + pub fn keyframes(&self) -> &[(u32, f64)] { + &self.keyframes + } +} + +fn compute_seek_time(keyframe_index: Option<&KeyframeIndex>, requested_time: f32) -> f32 { + if let Some(kf_index) = keyframe_index { + let fps = kf_index.fps(); + let target_frame = (requested_time as f64 * fps).round() as u32; + if let Some((_, keyframe_time)) = kf_index.nearest_keyframe_before(target_frame) { + return keyframe_time as f32; + } + } + requested_time +} + pub struct AVAssetReaderDecoder { path: PathBuf, pixel_format: cv::PixelFormat, @@ -18,10 +184,41 @@ pub struct AVAssetReaderDecoder { reader: R, width: u32, height: u32, + keyframe_index: Option, + current_position_secs: f32, } impl AVAssetReaderDecoder { pub fn new(path: PathBuf, tokio_handle: TokioHandle) -> Result { + Self::new_at_position(path, tokio_handle, 0.0) + } + + pub fn new_at_position( + path: PathBuf, + tokio_handle: TokioHandle, + start_time: f32, + ) -> Result { + let keyframe_index = match KeyframeIndex::build(&path) { + Ok(index) => Some(index), + Err(e) => { + tracing::warn!( + path = %path.display(), + error = %e, + "Failed to build keyframe index, seeking may be slower" + ); + None + } + }; + + Self::new_with_keyframe_index(path, tokio_handle, start_time, keyframe_index) + } + + pub fn new_with_keyframe_index( + path: PathBuf, + tokio_handle: TokioHandle, + start_time: f32, + keyframe_index: Option, + ) -> Result { let (pixel_format, width, height) = { let input = ffmpeg::format::input(&path).unwrap(); @@ -44,8 +241,16 @@ impl AVAssetReaderDecoder { ) }; - let (track_output, reader) = - Self::get_reader_track_output(&path, 0.0, &tokio_handle, pixel_format, width, height)?; + let seek_time = compute_seek_time(keyframe_index.as_ref(), start_time); + + let (track_output, reader) = Self::get_reader_track_output( + &path, + seek_time, + &tokio_handle, + pixel_format, + width, + height, + )?; Ok(Self { path, @@ -55,25 +260,56 @@ impl AVAssetReaderDecoder { reader, width, height, + keyframe_index, + current_position_secs: seek_time, }) } pub fn reset(&mut self, requested_time: f32) -> Result<(), String> { self.reader.cancel_reading(); + + let seek_time = compute_seek_time(self.keyframe_index.as_ref(), requested_time); + (self.track_output, self.reader) = Self::get_reader_track_output( &self.path, - requested_time, + seek_time, &self.tokio_handle, self.pixel_format, self.width, self.height, )?; + self.current_position_secs = seek_time; + Ok(()) } + pub fn current_position_secs(&self) -> f32 { + self.current_position_secs + } + + pub fn update_position(&mut self, position_secs: f32) { + self.current_position_secs = position_secs; + } + + pub fn path(&self) -> &PathBuf { + &self.path + } + + pub fn pixel_format(&self) -> cv::PixelFormat { + self.pixel_format + } + + pub fn take_keyframe_index(&mut self) -> Option { + self.keyframe_index.take() + } + + pub fn keyframe_index(&self) -> Option<&KeyframeIndex> { + self.keyframe_index.as_ref() + } + fn get_reader_track_output( - path: &PathBuf, + path: &Path, time: f32, handle: &TokioHandle, pixel_format: cv::PixelFormat, diff --git a/crates/video-decode/src/lib.rs b/crates/video-decode/src/lib.rs index ae408bc902..d14bd82696 100644 --- a/crates/video-decode/src/lib.rs +++ b/crates/video-decode/src/lib.rs @@ -5,7 +5,7 @@ pub mod ffmpeg; pub mod media_foundation; #[cfg(target_os = "macos")] -pub use avassetreader::AVAssetReaderDecoder; +pub use avassetreader::{AVAssetReaderDecoder, KeyframeIndex}; pub use ffmpeg::FFmpegDecoder; #[cfg(target_os = "windows")] pub use media_foundation::{ diff --git a/packages/ui-solid/src/auto-imports.d.ts b/packages/ui-solid/src/auto-imports.d.ts index 2e7c41dfb8..b44723b447 100644 --- a/packages/ui-solid/src/auto-imports.d.ts +++ b/packages/ui-solid/src/auto-imports.d.ts @@ -6,6 +6,7 @@ // biome-ignore lint: disable export {} declare global { + const IconCapArrowLeft: typeof import('~icons/cap/arrow-left.jsx')['default'] const IconCapArrows: typeof import('~icons/cap/arrows.jsx')['default'] const IconCapAudioOn: typeof import('~icons/cap/audio-on.jsx')['default'] const IconCapAuto: typeof import('~icons/cap/auto.jsx')['default'] @@ -26,6 +27,7 @@ declare global { const IconCapCursorWindows: typeof import('~icons/cap/cursor-windows.jsx')['default'] const IconCapEnlarge: typeof import('~icons/cap/enlarge.jsx')['default'] const IconCapFile: typeof import('~icons/cap/file.jsx')['default'] + const IconCapFilm: typeof import('~icons/cap/film.jsx')['default'] const IconCapFilmCut: typeof import('~icons/cap/film-cut.jsx')['default'] const IconCapGauge: typeof import('~icons/cap/gauge.jsx')['default'] const IconCapGear: typeof import('~icons/cap/gear.jsx')['default'] @@ -62,11 +64,13 @@ declare global { const IconCapStopCircle: typeof import('~icons/cap/stop-circle.jsx')['default'] const IconCapTrash: typeof import('~icons/cap/trash.jsx')['default'] const IconCapUndo: typeof import('~icons/cap/undo.jsx')['default'] + const IconCapUpload: typeof import('~icons/cap/upload.jsx')['default'] const IconCapX: typeof import('~icons/cap/x.jsx')['default'] const IconCapZoomIn: typeof import('~icons/cap/zoom-in.jsx')['default'] const IconCapZoomOut: typeof import('~icons/cap/zoom-out.jsx')['default'] const IconHugeiconsEaseCurveControlPoints: typeof import('~icons/hugeicons/ease-curve-control-points.jsx')['default'] const IconLucideAlertTriangle: typeof import('~icons/lucide/alert-triangle.jsx')['default'] + const IconLucideArrowLeft: typeof import('~icons/lucide/arrow-left.jsx')['default'] const IconLucideBell: typeof import('~icons/lucide/bell.jsx')['default'] const IconLucideBoxSelect: typeof import('~icons/lucide/box-select.jsx')['default'] const IconLucideBug: typeof import('~icons/lucide/bug.jsx')['default'] @@ -78,13 +82,16 @@ declare global { const IconLucideEyeOff: typeof import('~icons/lucide/eye-off.jsx')['default'] const IconLucideFastForward: typeof import('~icons/lucide/fast-forward.jsx')['default'] const IconLucideFolder: typeof import('~icons/lucide/folder.jsx')['default'] + const IconLucideGauge: typeof import('~icons/lucide/gauge.jsx')['default'] const IconLucideGift: typeof import('~icons/lucide/gift.jsx')['default'] const IconLucideHardDrive: typeof import('~icons/lucide/hard-drive.jsx')['default'] const IconLucideImage: typeof import('~icons/lucide/image.jsx')['default'] + const IconLucideInfo: typeof import('~icons/lucide/info.jsx')['default'] const IconLucideLayout: typeof import('~icons/lucide/layout.jsx')['default'] const IconLucideLoader2: typeof import('~icons/lucide/loader2.jsx')['default'] const IconLucideLoaderCircle: typeof import('~icons/lucide/loader-circle.jsx')['default'] const IconLucideMaximize: typeof import('~icons/lucide/maximize.jsx')['default'] + const IconLucideMaximize2: typeof import('~icons/lucide/maximize2.jsx')['default'] const IconLucideMessageSquarePlus: typeof import('~icons/lucide/message-square-plus.jsx')['default'] const IconLucideMicOff: typeof import('~icons/lucide/mic-off.jsx')['default'] const IconLucideMonitor: typeof import('~icons/lucide/monitor.jsx')['default'] @@ -94,6 +101,7 @@ declare global { const IconLucideRotateCcw: typeof import('~icons/lucide/rotate-ccw.jsx')['default'] const IconLucideSave: typeof import('~icons/lucide/save.jsx')['default'] const IconLucideSearch: typeof import('~icons/lucide/search.jsx')['default'] + const IconLucideSparkles: typeof import('~icons/lucide/sparkles.jsx')['default'] const IconLucideSquarePlay: typeof import('~icons/lucide/square-play.jsx')['default'] const IconLucideTimer: typeof import('~icons/lucide/timer.jsx')['default'] const IconLucideType: typeof import('~icons/lucide/type.jsx')['default'] @@ -101,6 +109,7 @@ declare global { const IconLucideVideo: typeof import('~icons/lucide/video.jsx')['default'] const IconLucideVolume2: typeof import('~icons/lucide/volume2.jsx')['default'] const IconLucideX: typeof import('~icons/lucide/x.jsx')['default'] + const IconLucideZap: typeof import('~icons/lucide/zap.jsx')['default'] const IconPhMonitorBold: typeof import('~icons/ph/monitor-bold.jsx')['default'] const IconPhRecordFill: typeof import('~icons/ph/record-fill.jsx')['default'] const IconPhWarningBold: typeof import('~icons/ph/warning-bold.jsx')['default']