diff --git a/.claude/agents/coderabbit-pr-reviewer.md b/.claude/agents/coderabbit-pr-reviewer.md new file mode 100644 index 0000000000..1bf03b3c3a --- /dev/null +++ b/.claude/agents/coderabbit-pr-reviewer.md @@ -0,0 +1,117 @@ +--- +name: coderabbit-pr-reviewer +description: Use this agent when you need to automatically implement CodeRabbit PR review suggestions from a GitHub pull request. This agent fetches review comments from the GitHub API, parses CodeRabbit's AI agent instructions, and systematically applies the suggested fixes while respecting project conventions.\n\nExamples:\n\n\nContext: User wants to implement CodeRabbit suggestions from a specific PR\nuser: "Implement the CodeRabbit suggestions from PR #1459"\nassistant: "I'll use the coderabbit-pr-reviewer agent to fetch and implement the CodeRabbit suggestions from PR #1459"\n\nSince the user wants to implement CodeRabbit suggestions, use the coderabbit-pr-reviewer agent to handle the complete workflow of fetching, parsing, and implementing the suggestions.\n\n\n\n\nContext: User mentions CodeRabbit review comments need to be addressed\nuser: "There are some CodeRabbit review comments on the PR that need fixing"\nassistant: "I'll launch the coderabbit-pr-reviewer agent to systematically implement the CodeRabbit review suggestions"\n\nThe user is referencing CodeRabbit review comments that need implementation. Use the coderabbit-pr-reviewer agent to handle this workflow.\n\n\n\n\nContext: User wants to address automated code review feedback\nuser: "Can you fix the issues that CodeRabbit found in CapSoftware/Cap pull request 1500?"\nassistant: "I'll use the coderabbit-pr-reviewer agent to fetch the CodeRabbit comments from PR #1500 in CapSoftware/Cap and implement the suggested fixes"\n\nThe user explicitly mentions CodeRabbit and a specific PR. Use the coderabbit-pr-reviewer agent to process these suggestions.\n\n +model: opus +color: red +--- + +You are an expert code review implementation agent specializing in automatically applying CodeRabbit PR review suggestions. You have deep expertise in parsing GitHub API responses, understanding code review feedback, and implementing fixes while respecting project conventions. + +## Your Mission + +You systematically fetch, parse, and implement CodeRabbit review suggestions from GitHub pull requests, adapting each fix to work within the project's existing architecture and dependencies. + +## Workflow + +### Phase 1: Fetch CodeRabbit Comments + +1. Determine the repository owner, repo name, and PR number from user input +2. Fetch PR review comments using the GitHub API: + - Endpoint: `GET https://api.github.com/repos/{owner}/{repo}/pulls/{pr_number}/comments` + - Filter for comments where `user.login == "coderabbitai[bot]"` +3. Extract key fields from each comment: + - `path`: The file to modify + - `line` or `original_line`: The line number + - `body`: The full markdown comment with instructions + +### Phase 2: Parse Each Comment + +For each CodeRabbit comment: + +1. **Extract the AI Agent Instructions** + - Look for the section: `
šŸ¤– Prompt for AI Agents` + - Parse the specific instructions within this block + +2. **Extract the Suggested Fix** + - Look for the section: `
šŸ”§ Suggested fix` + - Parse the diff blocks showing old vs new code + +3. **Understand the Issue Context** + - Note the issue type (āš ļø Potential issue, šŸ“Œ Major, etc.) + - Read the description explaining why the change is needed + +### Phase 3: Implement Each Fix + +For each suggestion: + +1. **Read Context** + - Open the target file at the specified line + - Read surrounding context (Ā±10 lines) + - Check the project's `Cargo.toml` or `package.json` for available dependencies + +2. **Adapt the Fix** + - Apply the suggested diff + - If suggested imports/crates don't exist, use alternatives: + - `tracing::warn!` ā†’ `eprintln!` (if tracing unavailable) + - `tracing::error!` ā†’ `eprintln!` (if tracing unavailable) + - `anyhow::Error` ā†’ `Box` (if anyhow unavailable) + - Respect project conventions (especially the NO COMMENTS rule for this codebase) + +3. **Common Fix Patterns** + - Silent Result handling: Replace `let _ = result` with `if let Err(e) = result { warn!(...) }` + - Panic prevention: Replace `panic!()` with warning logs and graceful handling + - Missing flush calls: Add explicit flush before returns + - UTF-8 safety: Use `.chars().take()` instead of byte slicing + - Platform handling: Add cfg-based platform branches + +### Phase 4: Validate Changes + +After implementing all fixes: + +1. **Format Code** + - Rust: `cargo fmt --all` + - TypeScript: `pnpm format` + +2. **Check Compilation** + - Rust: `cargo check -p affected_crate` + - TypeScript: `pnpm typecheck` + +3. **Lint Check** + - Rust: `cargo clippy` + - TypeScript: `pnpm lint` + +## Critical Rules + +1. **Never add code comments** - This project forbids all forms of comments. Code must be self-explanatory through naming, types, and structure. + +2. **Verify dependencies exist** before using them. Check Cargo.toml/package.json first. + +3. **Preserve existing code style** - Match the patterns used in surrounding code. + +4. **Skip conflicting suggestions** - If a CodeRabbit suggestion conflicts with project rules (like adding comments), skip it and report to the user. + +5. **Report unresolvable issues** - Some suggestions may require manual review. Document these clearly. + +## Output Format + +After completing implementation, provide: + +1. **Summary of Changes** + - List each file modified + - Brief description of each fix applied + +2. **Skipped Suggestions** + - Any suggestions that couldn't be implemented automatically + - Reason for skipping + +3. **Validation Results** + - Formatting status + - Compilation status + - Any remaining warnings or errors + +## Error Handling + +- If GitHub API fails: Report the error and suggest checking authentication or rate limits +- If a file doesn't exist: Skip that suggestion and note it in the report +- If compilation fails after a fix: Attempt to diagnose, or revert and report for manual review +- If no CodeRabbit comments found: Inform the user and suggest verifying the PR number diff --git a/.claude/settings.local.json b/.claude/settings.local.json index a9944f0f3d..b83de1a96a 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -42,7 +42,15 @@ "Bash(cargo tree:*)", "WebFetch(domain:github.com)", "WebFetch(domain:docs.rs)", - "WebFetch(domain:gix.github.io)" + "WebFetch(domain:gix.github.io)", + "Bash(cargo clean:*)", + "Bash(cargo test:*)", + "Bash(powershell -Command \"[System.Environment]::OSVersion.Version.ToString()\")", + "Bash(cargo build:*)", + "Bash(gh api:*)", + "Bash(curl:*)", + "Bash(node -e:*)", + "Bash(findstr:*)" ], "deny": [], "ask": [] diff --git a/Cargo.lock b/Cargo.lock index 97bceb9bfa..4aa78c2b5f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1122,6 +1122,7 @@ version = "0.1.0" dependencies = [ "cap-mediafoundation-utils", "inquire", + "parking_lot", "thiserror 1.0.69", "tracing", "windows 0.60.0", @@ -1327,6 +1328,7 @@ dependencies = [ name = "cap-enc-ffmpeg" version = "0.1.0" dependencies = [ + "cap-frame-converter", "cap-media-info", "ffmpeg-next", "serde", @@ -1358,6 +1360,7 @@ dependencies = [ "scap-direct3d", "scap-targets", "thiserror 1.0.69", + "tracing", "windows 0.60.0", "windows-core 0.60.1", "windows-numerics 0.2.0", @@ -7724,6 +7727,7 @@ dependencies = [ "scap-ffmpeg", "scap-targets", "thiserror 1.0.69", + "tracing", "windows 0.60.0", "windows-numerics 0.2.0", "workspace-hack", diff --git a/apps/desktop/src-tauri/src/general_settings.rs b/apps/desktop/src-tauri/src/general_settings.rs index 4c1373d43c..e970af964d 100644 --- a/apps/desktop/src-tauri/src/general_settings.rs +++ b/apps/desktop/src-tauri/src/general_settings.rs @@ -72,12 +72,11 @@ pub struct GeneralSettingsStore { pub hide_dock_icon: bool, #[serde(default)] pub auto_create_shareable_link: bool, - #[serde(default = "true_b")] + #[serde(default = "default_true")] pub enable_notifications: bool, #[serde(default)] pub disable_auto_open_links: bool, - // first launch: store won't exist so show startup - #[serde(default = "true_b")] + #[serde(default = "default_true")] pub has_completed_startup: bool, #[serde(default)] pub theme: AppTheme, @@ -192,7 +191,7 @@ impl Default for GeneralSettingsStore { delete_instant_recordings_after_upload: false, instant_mode_max_resolution: 1920, default_project_name_template: None, - crash_recovery_recording: false, + crash_recovery_recording: true, } } } @@ -206,10 +205,6 @@ pub enum AppTheme { Dark, } -fn true_b() -> bool { - true -} - impl GeneralSettingsStore { pub fn get(app: &AppHandle) -> Result, String> { match app.store("store").map(|s| s.get("general_settings")) { diff --git a/apps/desktop/src-tauri/src/lib.rs b/apps/desktop/src-tauri/src/lib.rs index f5c19546d3..e914955174 100644 --- a/apps/desktop/src-tauri/src/lib.rs +++ b/apps/desktop/src-tauri/src/lib.rs @@ -420,6 +420,12 @@ async fn upload_logs(app_handle: AppHandle) -> Result<(), String> { logging::upload_log_file(&app_handle).await } +#[tauri::command] +#[specta::specta] +fn get_system_diagnostics() -> cap_recording::diagnostics::SystemDiagnostics { + cap_recording::diagnostics::collect_diagnostics() +} + #[tauri::command] #[specta::specta] #[instrument(skip(app_handle, state))] @@ -1226,7 +1232,7 @@ async fn open_file_path(_app: AppHandle, path: PathBuf) -> Result<(), String> { Command::new("explorer") .args(["/select,", path_str]) .spawn() - .map_err(|e| format!("Failed to open folder: {}", e))?; + .map_err(|e| format!("Failed to open folder: {e}"))?; } #[cfg(target_os = "macos")] @@ -1248,7 +1254,7 @@ async fn open_file_path(_app: AppHandle, path: PathBuf) -> Result<(), String> { .ok_or("Invalid path")?, ) .spawn() - .map_err(|e| format!("Failed to open folder: {}", e))?; + .map_err(|e| format!("Failed to open folder: {e}"))?; } Ok(()) @@ -2337,6 +2343,7 @@ pub async fn run(recording_logging_handle: LoggingHandle, logs_dir: PathBuf) { set_camera_input, recording_settings::set_recording_mode, upload_logs, + get_system_diagnostics, recording::start_recording, recording::stop_recording, recording::pause_recording, diff --git a/apps/desktop/src-tauri/src/windows.rs b/apps/desktop/src-tauri/src/windows.rs index b36d554334..552eae2a64 100644 --- a/apps/desktop/src-tauri/src/windows.rs +++ b/apps/desktop/src-tauri/src/windows.rs @@ -272,15 +272,32 @@ impl ShowCapWindow { let monitor = app.primary_monitor()?.unwrap(); let window = match self { - Self::Setup => self - .window_builder(app, "/setup") - .resizable(false) - .maximized(false) - .center() - .focused(true) - .maximizable(false) - .shadow(true) - .build()?, + Self::Setup => { + let window = self + .window_builder(app, "/setup") + .inner_size(600.0, 600.0) + .min_inner_size(600.0, 600.0) + .resizable(false) + .maximized(false) + .center() + .focused(true) + .maximizable(false) + .shadow(true) + .build()?; + + #[cfg(windows)] + { + use tauri::LogicalSize; + if let Err(e) = window.set_size(LogicalSize::new(600.0, 600.0)) { + warn!("Failed to set Setup window size on Windows: {}", e); + } + if let Err(e) = window.center() { + warn!("Failed to center Setup window on Windows: {}", e); + } + } + + window + } Self::Main { init_target_mode } => { if !permissions::do_permissions_check(false).necessary_granted() { return Box::pin(Self::Setup.show(app)).await; @@ -417,7 +434,6 @@ impl ShowCapWindow { window } Self::Settings { page } => { - // Hide main window and target select overlays when settings window opens for (label, window) in app.webview_windows() { if let Ok(id) = CapWindowId::from_str(&label) && matches!( @@ -431,14 +447,30 @@ impl ShowCapWindow { } } - self.window_builder( - app, - format!("/settings/{}", page.clone().unwrap_or_default()), - ) - .resizable(true) - .maximized(false) - .center() - .build()? + let window = self + .window_builder( + app, + format!("/settings/{}", page.clone().unwrap_or_default()), + ) + .inner_size(600.0, 465.0) + .min_inner_size(600.0, 465.0) + .resizable(true) + .maximized(false) + .center() + .build()?; + + #[cfg(windows)] + { + use tauri::LogicalSize; + if let Err(e) = window.set_size(LogicalSize::new(600.0, 465.0)) { + warn!("Failed to set Settings window size on Windows: {}", e); + } + if let Err(e) = window.center() { + warn!("Failed to center Settings window on Windows: {}", e); + } + } + + window } Self::Editor { .. } => { if let Some(main) = CapWindowId::Main.get(app) { @@ -448,11 +480,26 @@ impl ShowCapWindow { let _ = camera.close(); }; - self.window_builder(app, "/editor") + let window = self + .window_builder(app, "/editor") .maximizable(true) - .inner_size(1240.0, 800.0) + .inner_size(1275.0, 800.0) + .min_inner_size(1275.0, 800.0) .center() - .build()? + .build()?; + + #[cfg(windows)] + { + use tauri::LogicalSize; + if let Err(e) = window.set_size(LogicalSize::new(1275.0, 800.0)) { + warn!("Failed to set Editor window size on Windows: {}", e); + } + if let Err(e) = window.center() { + warn!("Failed to center Editor window on Windows: {}", e); + } + } + + window } Self::ScreenshotEditor { path: _ } => { if let Some(main) = CapWindowId::Main.get(app) { @@ -462,35 +509,66 @@ impl ShowCapWindow { let _ = camera.close(); }; - self.window_builder(app, "/screenshot-editor") + let window = self + .window_builder(app, "/screenshot-editor") .maximizable(true) .inner_size(1240.0, 800.0) + .min_inner_size(800.0, 600.0) .center() - .build()? + .build()?; + + #[cfg(windows)] + { + use tauri::LogicalSize; + if let Err(e) = window.set_size(LogicalSize::new(1240.0, 800.0)) { + warn!( + "Failed to set ScreenshotEditor window size on Windows: {}", + e + ); + } + if let Err(e) = window.center() { + warn!("Failed to center ScreenshotEditor window on Windows: {}", e); + } + } + + window } Self::Upgrade => { - // Hide main window when upgrade window opens if let Some(main) = CapWindowId::Main.get(app) { let _ = main.hide(); } - let mut builder = self + let window = self .window_builder(app, "/upgrade") + .inner_size(950.0, 850.0) + .min_inner_size(950.0, 850.0) .resizable(false) .focused(true) .always_on_top(true) .maximized(false) .shadow(true) - .center(); + .center() + .build()?; - builder.build()? + #[cfg(windows)] + { + use tauri::LogicalSize; + if let Err(e) = window.set_size(LogicalSize::new(950.0, 850.0)) { + warn!("Failed to set Upgrade window size on Windows: {}", e); + } + if let Err(e) = window.center() { + warn!("Failed to center Upgrade window on Windows: {}", e); + } + } + + window } Self::ModeSelect => { if let Some(main) = CapWindowId::Main.get(app) { let _ = main.hide(); } - let mut builder = self + let window = self .window_builder(app, "/mode-select") .inner_size(580.0, 340.0) .min_inner_size(580.0, 340.0) @@ -499,9 +577,21 @@ impl ShowCapWindow { .maximizable(false) .center() .focused(true) - .shadow(true); + .shadow(true) + .build()?; - builder.build()? + #[cfg(windows)] + { + use tauri::LogicalSize; + if let Err(e) = window.set_size(LogicalSize::new(580.0, 340.0)) { + warn!("Failed to set ModeSelect window size on Windows: {}", e); + } + if let Err(e) = window.center() { + warn!("Failed to center ModeSelect window on Windows: {}", e); + } + } + + window } Self::Camera => { const WINDOW_SIZE: f64 = 230.0 * 2.0; diff --git a/apps/desktop/src/routes/(window-chrome)/settings/experimental.tsx b/apps/desktop/src/routes/(window-chrome)/settings/experimental.tsx index 0e6ba2a7de..3819fdca2b 100644 --- a/apps/desktop/src/routes/(window-chrome)/settings/experimental.tsx +++ b/apps/desktop/src/routes/(window-chrome)/settings/experimental.tsx @@ -27,7 +27,7 @@ function Inner(props: { initialStore: GeneralSettingsStore | null }) { enableNewRecordingFlow: true, autoZoomOnClicks: false, custom_cursor_capture2: true, - crashRecoveryRecording: false, + crashRecoveryRecording: true, }, ); diff --git a/apps/desktop/src/routes/(window-chrome)/settings/feedback.tsx b/apps/desktop/src/routes/(window-chrome)/settings/feedback.tsx index f81059faac..ae984036cf 100644 --- a/apps/desktop/src/routes/(window-chrome)/settings/feedback.tsx +++ b/apps/desktop/src/routes/(window-chrome)/settings/feedback.tsx @@ -2,10 +2,10 @@ import { Button } from "@cap/ui-solid"; import { action, useAction, useSubmission } from "@solidjs/router"; import { getVersion } from "@tauri-apps/api/app"; import { type as ostype } from "@tauri-apps/plugin-os"; -import { createSignal } from "solid-js"; +import { createResource, createSignal, For, Show } from "solid-js"; import toast from "solid-toast"; -import { commands } from "~/utils/tauri"; +import { commands, type SystemDiagnostics } from "~/utils/tauri"; import { apiClient, protectedHeaders } from "~/utils/web-api"; const sendFeedbackAction = action(async (feedback: string) => { @@ -18,9 +18,19 @@ const sendFeedbackAction = action(async (feedback: string) => { return response.body; }); +async function fetchDiagnostics(): Promise { + try { + return await commands.getSystemDiagnostics(); + } catch (e) { + console.error("Failed to fetch diagnostics:", e); + return null; + } +} + export default function FeedbackTab() { const [feedback, setFeedback] = createSignal(""); const [uploadingLogs, setUploadingLogs] = createSignal(false); + const [diagnostics] = createResource(fetchDiagnostics); const submission = useSubmission(sendFeedbackAction); const sendFeedback = useAction(sendFeedbackAction); @@ -107,6 +117,92 @@ export default function FeedbackTab() { {uploadingLogs() ? "Uploading..." : "Upload Logs"} + +
+

+ System Information +

+ + Loading system information... +

+ } + > + {(diag) => ( +
+ + {(ver) => ( +
+

Operating System

+

+ {ver().displayName} +

+
+ )} + + + + {(gpu) => ( +
+

Graphics

+

+ {gpu().description} ({gpu().vendor},{" "} + {gpu().dedicatedVideoMemoryMb} MB VRAM) +

+
+ )} + + +
+

Capture Support

+
+ + Graphics Capture:{" "} + {diag().graphicsCaptureSupported + ? "Supported" + : "Not Supported"} + + + D3D11 Video:{" "} + {diag().d3D11VideoProcessorAvailable + ? "Available" + : "Unavailable"} + +
+
+ + 0}> +
+

Available Encoders

+
+ + {(encoder) => ( + + {encoder} + + )} + +
+
+ +
+ )} + +
diff --git a/apps/desktop/src/utils/tauri.ts b/apps/desktop/src/utils/tauri.ts index 88b4042ec5..f78bc7845a 100644 --- a/apps/desktop/src/utils/tauri.ts +++ b/apps/desktop/src/utils/tauri.ts @@ -17,6 +17,9 @@ async setRecordingMode(mode: RecordingMode) : Promise { async uploadLogs() : Promise { return await TAURI_INVOKE("upload_logs"); }, +async getSystemDiagnostics() : Promise { + return await TAURI_INVOKE("get_system_diagnostics"); +}, async startRecording(inputs: StartRecordingInputs) : Promise { return await TAURI_INVOKE("start_recording", { inputs }); }, @@ -355,6 +358,7 @@ uploadProgressEvent: "upload-progress-event" /** user-defined types **/ +export type AllGpusInfo = { gpus: GpuInfoDiag[]; primaryGpuIndex: number | null; isMultiGpuSystem: boolean; hasDiscreteGpu: boolean } export type Annotation = { id: string; type: AnnotationType; x: number; y: number; width: number; height: number; strokeColor: string; strokeWidth: number; fillColor: string; opacity: number; rotation: number; text: string | null; maskType?: MaskType | null; maskLevel?: number | null } export type AnnotationType = "arrow" | "circle" | "rectangle" | "text" | "mask" export type AppTheme = "system" | "light" | "dark" @@ -420,6 +424,7 @@ quality: number | null; * Whether to prioritize speed over quality (default: false) */ fast: boolean | null } +export type GpuInfoDiag = { vendor: string; description: string; dedicatedVideoMemoryMb: number; adapterIndex: number; isSoftwareAdapter: boolean; isBasicRenderDriver: boolean; supportsHardwareEncoding: boolean } export type HapticPattern = "alignment" | "levelChange" | "generic" export type HapticPerformanceTime = "default" | "now" | "drawCompleted" export type Hotkey = { code: string; meta: boolean; ctrl: boolean; alt: boolean; shift: boolean } @@ -474,6 +479,7 @@ export type RecordingStatus = "pending" | "recording" export type RecordingStopped = null export type RecordingTargetMode = "display" | "window" | "area" export type RenderFrameEvent = { frame_number: number; fps: number; resolution_base: XY } +export type RenderingStatus = { isUsingSoftwareRendering: boolean; isUsingBasicRenderDriver: boolean; hardwareEncodingAvailable: boolean; warningMessage: string | null } export type RequestOpenRecordingPicker = { target_mode: RecordingTargetMode | null } export type RequestOpenSettings = { page: string } export type RequestScreenCapturePrewarm = { force?: boolean } @@ -494,6 +500,7 @@ export type StartRecordingInputs = { capture_target: ScreenCaptureTarget; captur export type StereoMode = "stereo" | "monoL" | "monoR" export type StudioRecordingMeta = { segment: SingleSegment } | { inner: MultipleSegments } export type StudioRecordingStatus = { status: "InProgress" } | { status: "NeedsRemux" } | { status: "Failed"; error: string } | { status: "Complete" } +export type SystemDiagnostics = { windowsVersion: WindowsVersionInfo | null; gpuInfo: GpuInfoDiag | null; allGpus: AllGpusInfo | null; renderingStatus: RenderingStatus; availableEncoders: string[]; graphicsCaptureSupported: boolean; d3D11VideoProcessorAvailable: boolean } export type TargetUnderCursor = { display_id: DisplayId | null; window: WindowUnderCursor | null } export type TextSegment = { start: number; end: number; enabled?: boolean; content?: string; center?: XY; size?: XY; fontFamily?: string; fontSize?: number; fontWeight?: number; italic?: boolean; color?: string; fadeDuration?: number } export type TimelineConfiguration = { segments: TimelineSegment[]; zoomSegments: ZoomSegment[]; sceneSegments?: SceneSegment[]; maskSegments?: MaskSegment[]; textSegments?: TextSegment[] } @@ -510,6 +517,7 @@ export type VideoUploadInfo = { id: string; link: string; config: S3UploadMeta } export type WindowExclusion = { bundleIdentifier?: string | null; ownerName?: string | null; windowTitle?: string | null } export type WindowId = string export type WindowUnderCursor = { id: WindowId; app_name: string; bounds: LogicalBounds } +export type WindowsVersionInfo = { major: number; minor: number; build: number; displayName: string; meetsRequirements: boolean; isWindows11: boolean } export type XY = { x: T; y: T } export type ZoomMode = "auto" | { manual: { x: number; y: number } } export type ZoomSegment = { start: number; end: number; amount: number; mode: ZoomMode } diff --git a/crates/camera-directshow/examples/cli.rs b/crates/camera-directshow/examples/cli.rs index f154aa99a6..00161139dd 100644 --- a/crates/camera-directshow/examples/cli.rs +++ b/crates/camera-directshow/examples/cli.rs @@ -56,7 +56,7 @@ mod windows { return None; } - let video_info = &*media_type.video_info(); + let video_info = media_type.video_info(); let width = video_info.bmiHeader.biWidth; let height = video_info.bmiHeader.biHeight; @@ -114,7 +114,7 @@ mod windows { .start_capturing( &selected_format.media_type, Box::new(|frame| { - let data_length = unsafe { frame.sample.GetActualDataLength() }; + let data_length = frame.sample.GetActualDataLength(); println!( "Frame: data_length={data_length:?}, timestamp={:?}", frame.timestamp diff --git a/crates/camera-mediafoundation/Cargo.toml b/crates/camera-mediafoundation/Cargo.toml index 0bee6fe27d..36ee6fdd9d 100644 --- a/crates/camera-mediafoundation/Cargo.toml +++ b/crates/camera-mediafoundation/Cargo.toml @@ -7,6 +7,7 @@ license = "MIT" [dependencies] tracing.workspace = true thiserror.workspace = true +parking_lot = "0.12" workspace-hack = { version = "0.1", path = "../workspace-hack" } [target.'cfg(windows)'.dependencies] diff --git a/crates/camera-mediafoundation/src/lib.rs b/crates/camera-mediafoundation/src/lib.rs index e4335d9634..5e7610ac9a 100644 --- a/crates/camera-mediafoundation/src/lib.rs +++ b/crates/camera-mediafoundation/src/lib.rs @@ -2,6 +2,7 @@ #![allow(non_snake_case)] use cap_mediafoundation_utils::*; +use parking_lot::Mutex; use std::{ ffi::OsString, fmt::Display, @@ -9,10 +10,7 @@ use std::{ ops::{Deref, DerefMut}, os::windows::ffi::OsStringExt, slice::from_raw_parts, - sync::{ - Mutex, - mpsc::{Receiver, Sender, channel}, - }, + sync::mpsc::{Receiver, Sender, channel}, time::Duration, }; use tracing::error; @@ -227,7 +225,7 @@ impl Device { .SetUINT32(&MF_CAPTURE_ENGINE_USE_VIDEO_DEVICE_ONLY, 1) .map_err(StartCapturingError::ConfigureEngine)?; - println!("Initializing engine..."); + tracing::debug!("Initializing Media Foundation capture engine"); engine .Initialize( @@ -244,7 +242,7 @@ impl Device { )); }; - println!("Engine initialized."); + tracing::debug!("Media Foundation capture engine initialized"); let source = engine .GetSource() @@ -345,8 +343,11 @@ fn retry_on_invalid_request( ) -> windows_core::Result { let mut retry_count = 0; - const MAX_RETRIES: u32 = 100; - const RETRY_DELAY: Duration = Duration::from_millis(50); + const MAX_RETRIES: u32 = 50; + const INITIAL_DELAY_MS: u64 = 1; + const MAX_DELAY_MS: u64 = 50; + + let mut current_delay_ms = INITIAL_DELAY_MS; loop { match cb() { @@ -356,7 +357,8 @@ fn retry_on_invalid_request( return Err(e); } retry_count += 1; - std::thread::sleep(RETRY_DELAY); + std::thread::sleep(Duration::from_millis(current_delay_ms)); + current_delay_ms = (current_delay_ms * 2).min(MAX_DELAY_MS); } Err(e) => return Err(e), } @@ -467,18 +469,9 @@ pub struct VideoSample(IMFSample); impl VideoSample { pub fn bytes(&self) -> windows_core::Result> { unsafe { - let bytes = self.0.GetTotalLength()?; - let mut out = Vec::with_capacity(bytes as usize); - - let buffer_count = self.0.GetBufferCount()?; - for buffer_i in 0..buffer_count { - let buffer = self.0.GetBufferByIndex(buffer_i)?; - - let bytes = buffer.lock()?; - out.extend(&*bytes); - } - - Ok(out) + let buffer = self.0.ConvertToContiguousBuffer()?; + let locked = buffer.lock()?; + Ok(locked.to_vec()) } } } @@ -564,9 +557,7 @@ impl IMFCaptureEngineOnSampleCallback_Impl for VideoCallback_Impl { return Ok(()); }; - let Ok(mut callback) = self.sample_callback.lock() else { - return Ok(()); - }; + let mut callback = self.sample_callback.lock(); let sample_time = unsafe { sample.GetSampleTime() }?; diff --git a/crates/camera-windows/examples/cli.rs b/crates/camera-windows/examples/cli.rs index fbf898bc06..4725556844 100644 --- a/crates/camera-windows/examples/cli.rs +++ b/crates/camera-windows/examples/cli.rs @@ -69,5 +69,6 @@ mod windows { } } + #[allow(dead_code)] pub struct FormatSelection(VideoFormat); } diff --git a/crates/camera-windows/src/lib.rs b/crates/camera-windows/src/lib.rs index 58aa97349e..e3fac3c88a 100644 --- a/crates/camera-windows/src/lib.rs +++ b/crates/camera-windows/src/lib.rs @@ -21,12 +21,148 @@ const MF_VIDEO_FORMAT_P010: GUID = GUID::from_u128(0x30313050_0000_0010_8000_00a const MEDIASUBTYPE_Y800: GUID = GUID::from_u128(0x30303859_0000_0010_8000_00aa00389b71); +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DeviceCategory { + Physical, + Virtual, + CaptureCard, +} + +impl DeviceCategory { + pub fn is_virtual(&self) -> bool { + matches!(self, DeviceCategory::Virtual) + } + + pub fn is_capture_card(&self) -> bool { + matches!(self, DeviceCategory::CaptureCard) + } +} + +const VIRTUAL_CAMERA_PATTERNS: &[&str] = &[ + "obs", + "virtual", + "snap camera", + "manycam", + "xsplit", + "streamlabs", + "droidcam", + "iriun", + "epoccam", + "ndi", + "newtek", + "camtwist", + "mmhmm", + "chromacam", + "vtuber", + "prism live", + "camo", + "avatarify", + "facerig", +]; + +const CAPTURE_CARD_PATTERNS: &[&str] = &[ + "elgato", + "avermedia", + "magewell", + "blackmagic", + "decklink", + "intensity", + "ultrastudio", + "atomos", + "hauppauge", + "startech", + "j5create", + "razer ripsaw", + "pengo", + "evga xr1", + "nzxt signal", + "genki shadowcast", + "cam link", + "live gamer", + "game capture", +]; + +fn detect_device_category(name: &OsStr, model_id: Option<&str>) -> DeviceCategory { + let name_lower = name.to_string_lossy().to_lowercase(); + let model_lower = model_id.map(|m| m.to_lowercase()); + + let matches_pattern = |patterns: &[&str]| { + patterns.iter().any(|pattern| { + name_lower.contains(pattern) + || model_lower.as_ref().is_some_and(|m| m.contains(pattern)) + }) + }; + + if matches_pattern(CAPTURE_CARD_PATTERNS) { + DeviceCategory::CaptureCard + } else if matches_pattern(VIRTUAL_CAMERA_PATTERNS) { + DeviceCategory::Virtual + } else { + DeviceCategory::Physical + } +} + +#[derive(Debug, Clone)] +pub struct FormatPreference { + pub width: u32, + pub height: u32, + pub frame_rate: f32, + pub format_priority: Vec, +} + +impl FormatPreference { + pub fn new(width: u32, height: u32, frame_rate: f32) -> Self { + Self { + width, + height, + frame_rate, + format_priority: vec![ + PixelFormat::NV12, + PixelFormat::YUYV422, + PixelFormat::UYVY422, + PixelFormat::YUV420P, + PixelFormat::MJPEG, + PixelFormat::RGB32, + ], + } + } + + pub fn with_format_priority(mut self, priority: Vec) -> Self { + self.format_priority = priority; + self + } + + pub fn for_hardware_encoding() -> Self { + Self::new(1920, 1080, 30.0).with_format_priority(vec![ + PixelFormat::NV12, + PixelFormat::YUYV422, + PixelFormat::UYVY422, + PixelFormat::YUV420P, + ]) + } + + pub fn for_capture_card() -> Self { + Self::new(1920, 1080, 60.0).with_format_priority(vec![ + PixelFormat::NV12, + PixelFormat::YUYV422, + PixelFormat::UYVY422, + PixelFormat::P010, + ]) + } +} + +impl Default for FormatPreference { + fn default() -> Self { + Self::new(1920, 1080, 30.0) + } +} + #[derive(Clone)] pub struct VideoDeviceInfo { id: OsString, name: OsString, model_id: Option, - // formats: Vec, + category: DeviceCategory, inner: VideoDeviceInfoInner, } @@ -67,6 +203,104 @@ impl VideoDeviceInfo { self.model_id.as_deref() } + pub fn category(&self) -> DeviceCategory { + self.category + } + + pub fn is_virtual_camera(&self) -> bool { + self.category.is_virtual() + } + + pub fn is_capture_card(&self) -> bool { + self.category.is_capture_card() + } + + pub fn is_high_bandwidth(&self) -> bool { + if !self.is_capture_card() { + return false; + } + self.formats().iter().any(|f| { + let pixels = f.width() as u64 * f.height() as u64; + let fps = f.frame_rate() as u64; + pixels >= 3840 * 2160 && fps >= 30 + }) + } + + pub fn max_resolution(&self) -> Option<(u32, u32)> { + self.formats() + .iter() + .map(|f| (f.width(), f.height())) + .max_by_key(|(w, h)| (*w as u64) * (*h as u64)) + } + + pub fn find_best_format(&self, preference: &FormatPreference) -> Option { + let formats = self.formats(); + if formats.is_empty() { + return None; + } + + let target_pixels = preference.width as u64 * preference.height as u64; + + let score_format = |f: &VideoFormat| { + let format_priority = preference + .format_priority + .iter() + .position(|&pf| pf == f.pixel_format()) + .map(|pos| 1000 - pos as i32) + .unwrap_or(0); + + let pixels = f.width() as u64 * f.height() as u64; + let resolution_score = if pixels == target_pixels { + 500 + } else if pixels > target_pixels { + 400 - ((pixels - target_pixels) / 10000).min(300) as i32 + } else { + 300 - ((target_pixels - pixels) / 10000).min(200) as i32 + }; + + let fps_diff = (f.frame_rate() - preference.frame_rate).abs(); + let fps_score = 100 - (fps_diff * 10.0).min(100.0) as i32; + + format_priority + resolution_score + fps_score + }; + + formats.into_iter().max_by_key(score_format) + } + + pub fn find_format_with_fallback(&self, preference: &FormatPreference) -> Option { + if let Some(format) = self.find_best_format(preference) { + return Some(format); + } + + let fallback_formats = [ + PixelFormat::NV12, + PixelFormat::YUYV422, + PixelFormat::UYVY422, + PixelFormat::MJPEG, + PixelFormat::RGB32, + PixelFormat::YUV420P, + ]; + + let formats = self.formats(); + for fallback_pixel_format in fallback_formats { + if let Some(format) = formats + .iter() + .filter(|f| f.pixel_format() == fallback_pixel_format) + .max_by_key(|f| { + let res_score = (f.width() as i32).min(preference.width as i32) + + (f.height() as i32).min(preference.height as i32); + let fps_score = + (100.0 - (f.frame_rate() - preference.frame_rate).abs().min(100.0)) as i32; + res_score + fps_score + }) + { + return Some(format.clone()); + } + } + + formats.into_iter().next() + } + pub fn is_mf(&self) -> bool { matches!(self.inner, VideoDeviceInfoInner::MediaFoundation { .. }) } @@ -285,11 +519,13 @@ pub fn get_devices() -> Result, GetDevicesError> { let name = device.name()?; let id = device.id()?; let model_id = device.model_id(); + let category = detect_device_category(&name, model_id.as_deref()); Ok::<_, windows_core::Error>(VideoDeviceInfo { name, id, model_id, + category, inner: VideoDeviceInfoInner::MediaFoundation { device }, }) }) @@ -308,11 +544,13 @@ pub fn get_devices() -> Result, GetDevicesError> { let id = device.id()?; let name = device.name()?; let model_id = device.model_id(); + let category = detect_device_category(&name, model_id.as_deref()); Some(VideoDeviceInfo { name, id, model_id, + category, inner: VideoDeviceInfoInner::DirectShow(device), }) }) diff --git a/crates/cursor-info/examples/cli.rs b/crates/cursor-info/examples/cli.rs index 7bb589fab2..cee2b8fdf9 100644 --- a/crates/cursor-info/examples/cli.rs +++ b/crates/cursor-info/examples/cli.rs @@ -1,7 +1,10 @@ -use std::collections::HashMap; - -use cap_cursor_info::{CursorShape, CursorShapeMacOS}; +use cap_cursor_info::CursorShape; +#[cfg(target_os = "macos")] +use cap_cursor_info::CursorShapeMacOS; +#[cfg(target_os = "macos")] use sha2::{Digest, Sha256}; +#[cfg(target_os = "macos")] +use std::collections::HashMap; #[allow(unreachable_code)] fn main() { @@ -103,7 +106,7 @@ fn run() { // Try to convert HCURSOR to CursorShape using the TryFrom implementation match CursorShape::try_from(&cursor_info.hCursor) { Ok(cursor_shape) => { - println!("CursorShape: {}", cursor_shape); + println!("CursorShape: {cursor_shape}"); } Err(_) => { println!("Unknown cursor: {:?}", cursor_info.hCursor); diff --git a/crates/enc-ffmpeg/Cargo.toml b/crates/enc-ffmpeg/Cargo.toml index cf0244afac..0c010bceee 100644 --- a/crates/enc-ffmpeg/Cargo.toml +++ b/crates/enc-ffmpeg/Cargo.toml @@ -13,5 +13,8 @@ thiserror.workspace = true tracing.workspace = true workspace-hack = { version = "0.1", path = "../workspace-hack" } +[target.'cfg(target_os = "windows")'.dependencies] +cap-frame-converter = { path = "../frame-converter" } + [lints] workspace = true diff --git a/crates/enc-ffmpeg/src/video/h264.rs b/crates/enc-ffmpeg/src/video/h264.rs index 8f8614da4d..fa440c6f93 100644 --- a/crates/enc-ffmpeg/src/video/h264.rs +++ b/crates/enc-ffmpeg/src/video/h264.rs @@ -382,6 +382,46 @@ impl H264Encoder { } } +fn get_encoder_priority() -> &'static [&'static str] { + #[cfg(target_os = "macos")] + { + &[ + "h264_videotoolbox", + "h264_qsv", + "h264_nvenc", + "h264_amf", + "h264_mf", + "libx264", + ] + } + + #[cfg(target_os = "windows")] + { + use cap_frame_converter::{GpuVendor, detect_primary_gpu}; + + static ENCODER_PRIORITY_NVIDIA: &[&str] = + &["h264_nvenc", "h264_mf", "h264_qsv", "h264_amf", "libx264"]; + static ENCODER_PRIORITY_AMD: &[&str] = + &["h264_amf", "h264_mf", "h264_nvenc", "h264_qsv", "libx264"]; + static ENCODER_PRIORITY_INTEL: &[&str] = + &["h264_qsv", "h264_mf", "h264_nvenc", "h264_amf", "libx264"]; + static ENCODER_PRIORITY_DEFAULT: &[&str] = + &["h264_nvenc", "h264_qsv", "h264_amf", "h264_mf", "libx264"]; + + match detect_primary_gpu().map(|info| info.vendor) { + Some(GpuVendor::Nvidia) => ENCODER_PRIORITY_NVIDIA, + Some(GpuVendor::Amd) => ENCODER_PRIORITY_AMD, + Some(GpuVendor::Intel) => ENCODER_PRIORITY_INTEL, + _ => ENCODER_PRIORITY_DEFAULT, + } + } + + #[cfg(not(any(target_os = "macos", target_os = "windows")))] + { + &["libx264"] + } +} + fn get_codec_and_options( config: &VideoInfo, preset: H264Preset, @@ -395,18 +435,7 @@ fn get_codec_and_options( .max(1.0) as i32; let keyframe_interval_str = keyframe_interval.to_string(); - let encoder_priority: &[&str] = if cfg!(target_os = "macos") { - &[ - "h264_videotoolbox", - "h264_qsv", - "h264_nvenc", - "h264_amf", - "h264_mf", - "libx264", - ] - } else { - &["h264_nvenc", "h264_qsv", "h264_amf", "h264_mf", "libx264"] - }; + let encoder_priority = get_encoder_priority(); let mut encoders = Vec::new(); @@ -422,15 +451,21 @@ fn get_codec_and_options( options.set("realtime", "true"); } "h264_nvenc" => { - options.set("preset", "fast"); + options.set("preset", "p4"); + options.set("tune", "ll"); + options.set("rc", "vbr"); + options.set("spatial-aq", "1"); + options.set("temporal-aq", "1"); options.set("g", &keyframe_interval_str); } "h264_qsv" => { - options.set("preset", "fast"); + options.set("preset", "faster"); + options.set("look_ahead", "1"); options.set("g", &keyframe_interval_str); } "h264_amf" => { - options.set("quality", "speed"); + options.set("quality", "balanced"); + options.set("rc", "vbr_latency"); options.set("g", &keyframe_interval_str); } "h264_mf" => { diff --git a/crates/enc-ffmpeg/src/video/hevc.rs b/crates/enc-ffmpeg/src/video/hevc.rs new file mode 100644 index 0000000000..0f9e24ca96 --- /dev/null +++ b/crates/enc-ffmpeg/src/video/hevc.rs @@ -0,0 +1,507 @@ +use std::{thread, time::Duration}; + +use cap_media_info::{Pixel, VideoInfo}; +use ffmpeg::{ + Dictionary, + codec::{codec::Codec, context, encoder}, + format::{self}, + frame, + threading::Config, +}; +use tracing::{debug, error, trace}; + +use crate::base::EncoderBase; + +fn is_420(format: ffmpeg::format::Pixel) -> bool { + format + .descriptor() + .map(|desc| desc.log2_chroma_w() == 1 && desc.log2_chroma_h() == 1) + .unwrap_or(false) +} + +pub struct HevcEncoderBuilder { + bpp: f32, + input_config: VideoInfo, + preset: HevcPreset, + output_size: Option<(u32, u32)>, + external_conversion: bool, +} + +#[derive(Clone, Copy)] +pub enum HevcPreset { + Slow, + Medium, + Ultrafast, +} + +#[derive(thiserror::Error, Debug)] +pub enum HevcEncoderError { + #[error("{0:?}")] + FFmpeg(#[from] ffmpeg::Error), + #[error("Codec not found")] + CodecNotFound, + #[error("Pixel format {0:?} not supported")] + PixFmtNotSupported(Pixel), + #[error("Invalid output dimensions {width}x{height}; expected non-zero even width and height")] + InvalidOutputDimensions { width: u32, height: u32 }, +} + +impl HevcEncoderBuilder { + pub const QUALITY_BPP: f32 = 0.2; + + pub fn new(input_config: VideoInfo) -> Self { + Self { + input_config, + bpp: Self::QUALITY_BPP, + preset: HevcPreset::Ultrafast, + output_size: None, + external_conversion: false, + } + } + + pub fn with_preset(mut self, preset: HevcPreset) -> Self { + self.preset = preset; + self + } + + pub fn with_bpp(mut self, bpp: f32) -> Self { + self.bpp = bpp; + self + } + + pub fn with_output_size(mut self, width: u32, height: u32) -> Result { + if width == 0 || height == 0 { + return Err(HevcEncoderError::InvalidOutputDimensions { width, height }); + } + + self.output_size = Some((width, height)); + Ok(self) + } + + pub fn with_external_conversion(mut self) -> Self { + self.external_conversion = true; + self + } + + pub fn build( + self, + output: &mut format::context::Output, + ) -> Result { + let input_config = self.input_config; + let (output_width, output_height) = self + .output_size + .unwrap_or((input_config.width, input_config.height)); + + if output_width == 0 || output_height == 0 { + return Err(HevcEncoderError::InvalidOutputDimensions { + width: output_width, + height: output_height, + }); + } + + let candidates = get_codec_and_options(&input_config, self.preset); + if candidates.is_empty() { + return Err(HevcEncoderError::CodecNotFound); + } + + let mut last_error = None; + + for (codec, encoder_options) in candidates { + let codec_name = codec.name().to_string(); + + match Self::build_with_codec( + codec, + encoder_options, + &input_config, + output, + output_width, + output_height, + self.bpp, + self.external_conversion, + ) { + Ok(encoder) => { + debug!("Using HEVC encoder {}", codec_name); + return Ok(encoder); + } + Err(err) => { + debug!("HEVC encoder {} init failed: {:?}", codec_name, err); + last_error = Some(err); + } + } + } + + Err(last_error.unwrap_or(HevcEncoderError::CodecNotFound)) + } + + #[allow(clippy::too_many_arguments)] + fn build_with_codec( + codec: Codec, + encoder_options: Dictionary<'static>, + input_config: &VideoInfo, + output: &mut format::context::Output, + output_width: u32, + output_height: u32, + bpp: f32, + external_conversion: bool, + ) -> Result { + let encoder_supports_input_format = codec + .video() + .ok() + .and_then(|codec_video| codec_video.formats()) + .is_some_and(|mut formats| formats.any(|f| f == input_config.pixel_format)); + + let mut needs_pixel_conversion = false; + + let output_format = if encoder_supports_input_format { + input_config.pixel_format + } else { + needs_pixel_conversion = true; + let format = ffmpeg::format::Pixel::NV12; + if !external_conversion { + debug!( + "Converting from {:?} to {:?} for HEVC encoding", + input_config.pixel_format, format + ); + } + format + }; + + if is_420(output_format) + && (!output_width.is_multiple_of(2) || !output_height.is_multiple_of(2)) + { + return Err(HevcEncoderError::InvalidOutputDimensions { + width: output_width, + height: output_height, + }); + } + + let needs_scaling = + output_width != input_config.width || output_height != input_config.height; + + if needs_scaling && !external_conversion { + debug!( + "Scaling video frames for HEVC encoding from {}x{} to {}x{}", + input_config.width, input_config.height, output_width, output_height + ); + } + + let converter = if external_conversion { + debug!( + "External conversion enabled, skipping internal converter. Expected input: {:?} {}x{}", + output_format, output_width, output_height + ); + None + } else if needs_pixel_conversion || needs_scaling { + let flags = if needs_scaling { + ffmpeg::software::scaling::flag::Flags::BICUBIC + } else { + ffmpeg::software::scaling::flag::Flags::FAST_BILINEAR + }; + + match ffmpeg::software::scaling::Context::get( + input_config.pixel_format, + input_config.width, + input_config.height, + output_format, + output_width, + output_height, + flags, + ) { + Ok(context) => Some(context), + Err(e) => { + if needs_pixel_conversion { + error!( + "Failed to create converter from {:?} to {:?}: {:?}", + input_config.pixel_format, output_format, e + ); + return Err(HevcEncoderError::PixFmtNotSupported( + input_config.pixel_format, + )); + } + + return Err(HevcEncoderError::FFmpeg(e)); + } + } + } else { + None + }; + + let mut encoder_ctx = context::Context::new_with_codec(codec); + + let thread_count = thread::available_parallelism() + .map(|v| v.get()) + .unwrap_or(1); + encoder_ctx.set_threading(Config::count(thread_count)); + let mut encoder = encoder_ctx.encoder().video()?; + + encoder.set_width(output_width); + encoder.set_height(output_height); + encoder.set_format(output_format); + encoder.set_time_base(input_config.time_base); + encoder.set_frame_rate(Some(input_config.frame_rate)); + + let bitrate = get_bitrate( + output_width, + output_height, + input_config.frame_rate.0 as f32 / input_config.frame_rate.1.max(1) as f32, + bpp, + ); + + encoder.set_bit_rate(bitrate); + encoder.set_max_bit_rate(bitrate); + + let encoder = encoder.open_with(encoder_options)?; + + let mut output_stream = output.add_stream(codec)?; + let stream_index = output_stream.index(); + output_stream.set_time_base((1, HevcEncoder::TIME_BASE)); + output_stream.set_rate(input_config.frame_rate); + output_stream.set_parameters(&encoder); + + Ok(HevcEncoder { + base: EncoderBase::new(stream_index), + encoder, + converter, + output_format, + output_width, + output_height, + input_format: input_config.pixel_format, + input_width: input_config.width, + input_height: input_config.height, + }) + } +} + +pub struct HevcEncoder { + base: EncoderBase, + encoder: encoder::Video, + converter: Option, + output_format: format::Pixel, + output_width: u32, + output_height: u32, + input_format: format::Pixel, + input_width: u32, + input_height: u32, +} + +pub struct ConversionRequirements { + pub input_format: format::Pixel, + pub input_width: u32, + pub input_height: u32, + pub output_format: format::Pixel, + pub output_width: u32, + pub output_height: u32, + pub needs_conversion: bool, +} + +#[derive(thiserror::Error, Debug)] +pub enum QueueFrameError { + #[error("Converter: {0}")] + Converter(ffmpeg::Error), + #[error("Encode: {0}")] + Encode(ffmpeg::Error), +} + +impl HevcEncoder { + const TIME_BASE: i32 = 90000; + + pub fn builder(input_config: VideoInfo) -> HevcEncoderBuilder { + HevcEncoderBuilder::new(input_config) + } + + pub fn conversion_requirements(&self) -> ConversionRequirements { + let needs_conversion = self.input_format != self.output_format + || self.input_width != self.output_width + || self.input_height != self.output_height; + ConversionRequirements { + input_format: self.input_format, + input_width: self.input_width, + input_height: self.input_height, + output_format: self.output_format, + output_width: self.output_width, + output_height: self.output_height, + needs_conversion, + } + } + + pub fn queue_frame( + &mut self, + mut frame: frame::Video, + timestamp: Duration, + output: &mut format::context::Output, + ) -> Result<(), QueueFrameError> { + self.base + .update_pts(&mut frame, timestamp, &mut self.encoder); + + if let Some(converter) = &mut self.converter { + let pts = frame.pts(); + let mut converted = + frame::Video::new(self.output_format, self.output_width, self.output_height); + converter + .run(&frame, &mut converted) + .map_err(QueueFrameError::Converter)?; + converted.set_pts(pts); + frame = converted; + } + + self.base + .send_frame(&frame, output, &mut self.encoder) + .map_err(QueueFrameError::Encode)?; + + Ok(()) + } + + pub fn queue_preconverted_frame( + &mut self, + mut frame: frame::Video, + timestamp: Duration, + output: &mut format::context::Output, + ) -> Result<(), QueueFrameError> { + trace!( + "Encoding pre-converted frame: format={:?}, size={}x{}, expected={:?} {}x{}", + frame.format(), + frame.width(), + frame.height(), + self.output_format, + self.output_width, + self.output_height + ); + + self.base + .update_pts(&mut frame, timestamp, &mut self.encoder); + + self.base + .send_frame(&frame, output, &mut self.encoder) + .map_err(QueueFrameError::Encode)?; + + Ok(()) + } + + pub fn flush(&mut self, output: &mut format::context::Output) -> Result<(), ffmpeg::Error> { + self.base.process_eof(output, &mut self.encoder) + } +} + +fn get_encoder_priority() -> &'static [&'static str] { + #[cfg(target_os = "macos")] + { + &[ + "hevc_videotoolbox", + "hevc_qsv", + "hevc_nvenc", + "hevc_amf", + "hevc_mf", + "libx265", + ] + } + + #[cfg(target_os = "windows")] + { + use cap_frame_converter::{GpuVendor, detect_primary_gpu}; + + static ENCODER_PRIORITY_NVIDIA: &[&str] = + &["hevc_nvenc", "hevc_mf", "hevc_qsv", "hevc_amf", "libx265"]; + static ENCODER_PRIORITY_AMD: &[&str] = + &["hevc_amf", "hevc_mf", "hevc_nvenc", "hevc_qsv", "libx265"]; + static ENCODER_PRIORITY_INTEL: &[&str] = + &["hevc_qsv", "hevc_mf", "hevc_nvenc", "hevc_amf", "libx265"]; + static ENCODER_PRIORITY_DEFAULT: &[&str] = + &["hevc_nvenc", "hevc_qsv", "hevc_amf", "hevc_mf", "libx265"]; + + match detect_primary_gpu().map(|info| info.vendor) { + Some(GpuVendor::Nvidia) => ENCODER_PRIORITY_NVIDIA, + Some(GpuVendor::Amd) => ENCODER_PRIORITY_AMD, + Some(GpuVendor::Intel) => ENCODER_PRIORITY_INTEL, + _ => ENCODER_PRIORITY_DEFAULT, + } + } + + #[cfg(not(any(target_os = "macos", target_os = "windows")))] + { + &["libx265"] + } +} + +fn get_codec_and_options( + config: &VideoInfo, + preset: HevcPreset, +) -> Vec<(Codec, Dictionary<'static>)> { + let keyframe_interval_secs = 2; + let denominator = config.frame_rate.denominator(); + let frames_per_sec = config.frame_rate.numerator() as f64 + / if denominator == 0 { 1 } else { denominator } as f64; + let keyframe_interval = (keyframe_interval_secs as f64 * frames_per_sec) + .round() + .max(1.0) as i32; + let keyframe_interval_str = keyframe_interval.to_string(); + + let encoder_priority = get_encoder_priority(); + + let mut encoders = Vec::new(); + + for encoder_name in encoder_priority { + let Some(codec) = encoder::find_by_name(encoder_name) else { + continue; + }; + + let mut options = Dictionary::new(); + + match *encoder_name { + "hevc_videotoolbox" => { + options.set("realtime", "true"); + } + "hevc_nvenc" => { + options.set("preset", "p4"); + options.set("tune", "ll"); + options.set("rc", "vbr"); + options.set("spatial-aq", "1"); + options.set("temporal-aq", "1"); + options.set("tier", "main"); + options.set("g", &keyframe_interval_str); + } + "hevc_qsv" => { + options.set("preset", "faster"); + options.set("look_ahead", "1"); + options.set("g", &keyframe_interval_str); + } + "hevc_amf" => { + options.set("quality", "balanced"); + options.set("rc", "vbr_latency"); + options.set("g", &keyframe_interval_str); + } + "hevc_mf" => { + options.set("hw_encoding", "true"); + options.set("scenario", "4"); + options.set("quality", "1"); + options.set("g", &keyframe_interval_str); + } + "libx265" => { + options.set( + "preset", + match preset { + HevcPreset::Slow => "slow", + HevcPreset::Medium => "medium", + HevcPreset::Ultrafast => "ultrafast", + }, + ); + if let HevcPreset::Ultrafast = preset { + options.set("tune", "zerolatency"); + } + options.set("g", &keyframe_interval_str); + } + _ => {} + } + + encoders.push((codec, options)); + } + + encoders +} + +fn get_bitrate(width: u32, height: u32, frame_rate: f32, bpp: f32) -> usize { + let frame_rate_multiplier = ((frame_rate as f64 - 30.0).max(0.0) * 0.6) + 30.0; + let area = (width as f64) * (height as f64); + let pixels_per_second = area * frame_rate_multiplier; + + (pixels_per_second * bpp as f64) as usize +} diff --git a/crates/enc-ffmpeg/src/video/mod.rs b/crates/enc-ffmpeg/src/video/mod.rs index f61e796943..3429498fe7 100644 --- a/crates/enc-ffmpeg/src/video/mod.rs +++ b/crates/enc-ffmpeg/src/video/mod.rs @@ -1 +1,2 @@ pub mod h264; +pub mod hevc; diff --git a/crates/enc-mediafoundation/Cargo.toml b/crates/enc-mediafoundation/Cargo.toml index 7df3e9a3cb..cac0676628 100644 --- a/crates/enc-mediafoundation/Cargo.toml +++ b/crates/enc-mediafoundation/Cargo.toml @@ -7,6 +7,7 @@ edition = "2024" cap-media-info = { path = "../media-info" } futures.workspace = true thiserror.workspace = true +tracing.workspace = true workspace-hack = { version = "0.1", path = "../workspace-hack" } [target.'cfg(windows)'.dependencies] diff --git a/crates/enc-mediafoundation/examples/cli.rs b/crates/enc-mediafoundation/examples/cli.rs index f24f0a4fab..458bfb8cc6 100644 --- a/crates/enc-mediafoundation/examples/cli.rs +++ b/crates/enc-mediafoundation/examples/cli.rs @@ -18,7 +18,7 @@ mod win { Foundation::{Metadata::ApiInformation, TimeSpan}, Graphics::Capture::GraphicsCaptureSession, Win32::{ - Media::MediaFoundation::{self, MFSTARTUP_FULL, MFStartup}, + Media::MediaFoundation::{MFSTARTUP_FULL, MFStartup}, System::{ Diagnostics::Debug::{DebugBreak, IsDebuggerPresent}, Threading::GetCurrentProcessId, @@ -44,7 +44,7 @@ mod win { if wait_for_debugger { let pid = unsafe { GetCurrentProcessId() }; - println!("Waiting for a debugger to attach (PID: {})...", pid); + println!("Waiting for a debugger to attach (PID: {pid})..."); loop { if unsafe { IsDebuggerPresent().into() } { break; @@ -64,10 +64,7 @@ mod win { } if verbose { - println!( - "Using index \"{}\" and path \"{}\".", - display_index, output_path - ); + println!("Using index \"{display_index}\" and path \"{output_path}\"."); } let item = Display::primary() @@ -77,7 +74,7 @@ mod win { // Resolve encoding settings let resolution = item.Size()?; - let bit_rate = bit_rate * 1000000; + let _bit_rate = bit_rate * 1000000; // Start the recording { @@ -125,7 +122,7 @@ mod win { ) .unwrap(); - let output_path = std::env::current_dir().unwrap().join(output_path); + let _output_path = std::env::current_dir().unwrap().join(output_path); // let sample_writer = Arc::new(SampleWriter::new(output_path.as_path())?); @@ -216,7 +213,7 @@ mod win { } fn exit_with_error(message: &str) -> ! { - println!("{}", message); + println!("{message}"); std::process::exit(1); } @@ -289,6 +286,7 @@ mod win { } } + #[allow(dead_code)] mod hotkey { use std::sync::atomic::{AtomicI32, Ordering}; use windows::{ diff --git a/crates/enc-mediafoundation/src/video/h264.rs b/crates/enc-mediafoundation/src/video/h264.rs index 4fae2c1890..9611b8632d 100644 --- a/crates/enc-mediafoundation/src/video/h264.rs +++ b/crates/enc-mediafoundation/src/video/h264.rs @@ -3,9 +3,12 @@ use crate::{ mft::EncoderDevice, video::{NewVideoProcessorError, VideoProcessor}, }; -use std::sync::{ - Arc, - atomic::{AtomicBool, Ordering}, +use std::{ + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, + time::{Duration, Instant}, }; use windows::{ Foundation::TimeSpan, @@ -36,6 +39,89 @@ use windows::{ }; const MAX_CONSECUTIVE_EMPTY_SAMPLES: u8 = 20; +const MAX_INPUT_WITHOUT_OUTPUT: u32 = 30; +const MAX_PROCESS_INPUT_FAILURES: u32 = 5; +const ENCODER_OPERATION_TIMEOUT: Duration = Duration::from_secs(5); + +#[derive(Debug, Clone)] +pub struct EncoderHealthStatus { + pub inputs_without_output: u32, + pub consecutive_process_failures: u32, + pub total_frames_encoded: u64, + pub is_healthy: bool, + pub failure_reason: Option, +} + +#[derive(Debug, Clone)] +pub enum EncoderFailureReason { + Stalled, + ConsecutiveProcessFailures, + Timeout, + TooManyEmptySamples, +} + +struct EncoderHealthMonitor { + inputs_without_output: u32, + consecutive_process_failures: u32, + total_frames_encoded: u64, + last_output_time: Instant, +} + +impl EncoderHealthMonitor { + fn new() -> Self { + Self { + inputs_without_output: 0, + consecutive_process_failures: 0, + total_frames_encoded: 0, + last_output_time: Instant::now(), + } + } + + fn record_input(&mut self) { + self.inputs_without_output += 1; + } + + fn record_output(&mut self) { + self.inputs_without_output = 0; + self.consecutive_process_failures = 0; + self.total_frames_encoded += 1; + self.last_output_time = Instant::now(); + } + + fn record_process_failure(&mut self) { + self.consecutive_process_failures += 1; + } + + fn reset_process_failures(&mut self) { + self.consecutive_process_failures = 0; + } + + fn check_health(&self) -> EncoderHealthStatus { + let mut is_healthy = true; + let mut failure_reason = None; + + if self.inputs_without_output > MAX_INPUT_WITHOUT_OUTPUT { + is_healthy = false; + failure_reason = Some(EncoderFailureReason::Stalled); + } else if self.consecutive_process_failures >= MAX_PROCESS_INPUT_FAILURES { + is_healthy = false; + failure_reason = Some(EncoderFailureReason::ConsecutiveProcessFailures); + } else if self.last_output_time.elapsed() > ENCODER_OPERATION_TIMEOUT + && self.total_frames_encoded > 0 + { + is_healthy = false; + failure_reason = Some(EncoderFailureReason::Timeout); + } + + EncoderHealthStatus { + inputs_without_output: self.inputs_without_output, + consecutive_process_failures: self.consecutive_process_failures, + total_frames_encoded: self.total_frames_encoded, + is_healthy, + failure_reason, + } + } +} pub struct VideoEncoderOutputSample { sample: IMFSample, @@ -98,6 +184,36 @@ pub enum HandleNeedsInputError { ProcessInput(windows::core::Error), } +#[derive(Clone, Debug, thiserror::Error)] +pub enum EncoderRuntimeError { + #[error("Windows error: {0}")] + Windows(windows::core::Error), + #[error( + "Encoder unhealthy: {reason:?} (inputs_without_output={inputs_without_output}, process_failures={process_failures}, frames_encoded={frames_encoded})" + )] + EncoderUnhealthy { + reason: EncoderFailureReason, + inputs_without_output: u32, + process_failures: u32, + frames_encoded: u64, + }, +} + +impl EncoderRuntimeError { + pub fn should_fallback(&self) -> bool { + match self { + EncoderRuntimeError::Windows(_) => false, + EncoderRuntimeError::EncoderUnhealthy { .. } => true, + } + } +} + +impl From for EncoderRuntimeError { + fn from(err: windows::core::Error) -> Self { + EncoderRuntimeError::Windows(err) + } +} + unsafe impl Send for H264Encoder {} impl H264Encoder { @@ -385,12 +501,35 @@ impl H264Encoder { &self.output_type } + pub fn validate(&self) -> Result<(), NewVideoEncoderError> { + unsafe { + self.transform + .ProcessMessage(MFT_MESSAGE_COMMAND_FLUSH, 0) + .map_err(NewVideoEncoderError::EncoderTransform)?; + + self.transform + .ProcessMessage(MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, 0) + .map_err(NewVideoEncoderError::EncoderTransform)?; + + self.transform + .ProcessMessage(MFT_MESSAGE_NOTIFY_END_STREAMING, 0) + .map_err(NewVideoEncoderError::EncoderTransform)?; + + self.transform + .ProcessMessage(MFT_MESSAGE_COMMAND_FLUSH, 0) + .map_err(NewVideoEncoderError::EncoderTransform)?; + } + Ok(()) + } + pub fn run( &mut self, should_stop: Arc, mut get_frame: impl FnMut() -> windows::core::Result>, mut on_sample: impl FnMut(IMFSample) -> windows::core::Result<()>, - ) -> windows::core::Result<()> { + ) -> Result { + let mut health_monitor = EncoderHealthMonitor::new(); + unsafe { self.transform .ProcessMessage(MFT_MESSAGE_COMMAND_FLUSH, 0)?; @@ -399,33 +538,72 @@ impl H264Encoder { self.transform .ProcessMessage(MFT_MESSAGE_NOTIFY_START_OF_STREAM, 0)?; - let mut consecutive_empty_samples = 0; + let mut consecutive_empty_samples: u8 = 0; let mut should_exit = false; while !should_exit { + let health_status = health_monitor.check_health(); + if !health_status.is_healthy + && let Some(reason) = health_status.failure_reason + { + let _ = self.cleanup_encoder(); + return Err(EncoderRuntimeError::EncoderUnhealthy { + reason, + inputs_without_output: health_status.inputs_without_output, + process_failures: health_status.consecutive_process_failures, + frames_encoded: health_status.total_frames_encoded, + }); + } + let event = self.event_generator.GetEvent(MF_EVENT_FLAG_NONE)?; let event_type = MF_EVENT_TYPE(event.GetType()? as i32); match event_type { MediaFoundation::METransformNeedInput => { + health_monitor.record_input(); should_exit = true; if !should_stop.load(Ordering::SeqCst) && let Some((texture, timestamp)) = get_frame()? { - self.video_processor.process_texture(&texture)?; - let input_buffer = { - MFCreateDXGISurfaceBuffer( + let process_result = (|| -> windows::core::Result<()> { + self.video_processor.process_texture(&texture)?; + let input_buffer = MFCreateDXGISurfaceBuffer( &ID3D11Texture2D::IID, self.video_processor.output_texture(), 0, false, - )? - }; - let mf_sample = MFCreateSample()?; - mf_sample.AddBuffer(&input_buffer)?; - mf_sample.SetSampleTime(timestamp.Duration)?; - self.transform - .ProcessInput(self.input_stream_id, &mf_sample, 0)?; - should_exit = false; + )?; + let mf_sample = MFCreateSample()?; + mf_sample.AddBuffer(&input_buffer)?; + mf_sample.SetSampleTime(timestamp.Duration)?; + self.transform + .ProcessInput(self.input_stream_id, &mf_sample, 0)?; + Ok(()) + })(); + + match process_result { + Ok(()) => { + health_monitor.reset_process_failures(); + should_exit = false; + } + Err(_) => { + health_monitor.record_process_failure(); + let health_status = health_monitor.check_health(); + if !health_status.is_healthy + && let Some(reason) = health_status.failure_reason + { + let _ = self.cleanup_encoder(); + return Err(EncoderRuntimeError::EncoderUnhealthy { + reason, + inputs_without_output: health_status + .inputs_without_output, + process_failures: health_status + .consecutive_process_failures, + frames_encoded: health_status.total_frames_encoded, + }); + } + should_exit = false; + } + } } } MediaFoundation::METransformHaveOutput => { @@ -435,25 +613,24 @@ impl H264Encoder { ..Default::default() }; - // ProcessOutput may succeed but not populate pSample in some edge cases - // (e.g., hardware encoder transient failures, specific MFT implementations). - // This is a known contract violation by certain Media Foundation Transforms. - // We handle this gracefully by skipping the frame instead of panicking. let mut output_buffers = [output_buffer]; self.transform .ProcessOutput(0, &mut output_buffers, &mut status)?; - // Use the sample directly without cloning to prevent memory leaks if let Some(sample) = output_buffers[0].pSample.take() { consecutive_empty_samples = 0; + health_monitor.record_output(); on_sample(sample)?; } else { consecutive_empty_samples += 1; if consecutive_empty_samples > MAX_CONSECUTIVE_EMPTY_SAMPLES { - return Err(windows::core::Error::new( - windows::core::HRESULT(0), - "Too many consecutive empty samples", - )); + let _ = self.cleanup_encoder(); + return Err(EncoderRuntimeError::EncoderUnhealthy { + reason: EncoderFailureReason::TooManyEmptySamples, + inputs_without_output: health_monitor.inputs_without_output, + process_failures: health_monitor.consecutive_process_failures, + frames_encoded: health_monitor.total_frames_encoded, + }); } } } @@ -471,10 +648,23 @@ impl H264Encoder { .ProcessMessage(MFT_MESSAGE_COMMAND_FLUSH, 0)?; } - Ok(()) + Ok(health_monitor.check_health()) + } + + fn cleanup_encoder(&mut self) -> windows::core::Result<()> { + unsafe { + let _ = self + .transform + .ProcessMessage(MFT_MESSAGE_NOTIFY_END_OF_STREAM, 0); + let _ = self + .transform + .ProcessMessage(MFT_MESSAGE_NOTIFY_END_STREAMING, 0); + self.transform.ProcessMessage(MFT_MESSAGE_COMMAND_FLUSH, 0) + } } } fn calculate_bitrate(width: u32, height: u32, fps: u32, multiplier: f32) -> u32 { - ((width * height * ((fps - 30) / 2 + 30)) as f32 * multiplier) as u32 + let frame_rate_factor = (fps as f32 - 30.0).max(0.0) / 2.0 + 30.0; + (width as f32 * height as f32 * frame_rate_factor * multiplier) as u32 } diff --git a/crates/enc-mediafoundation/src/video/hevc.rs b/crates/enc-mediafoundation/src/video/hevc.rs new file mode 100644 index 0000000000..c99e476fa7 --- /dev/null +++ b/crates/enc-mediafoundation/src/video/hevc.rs @@ -0,0 +1,450 @@ +use crate::{ + media::{MFSetAttributeRatio, MFSetAttributeSize}, + mft::EncoderDevice, + video::{NewVideoProcessorError, VideoProcessor}, +}; +use std::sync::{ + Arc, + atomic::{AtomicBool, Ordering}, +}; +use windows::{ + Foundation::TimeSpan, + Graphics::SizeInt32, + Win32::{ + Foundation::E_NOTIMPL, + Graphics::{ + Direct3D11::{ID3D11Device, ID3D11Texture2D}, + Dxgi::Common::{DXGI_FORMAT, DXGI_FORMAT_NV12}, + }, + Media::MediaFoundation::{ + self, IMFAttributes, IMFDXGIDeviceManager, IMFMediaEventGenerator, IMFMediaType, + IMFSample, IMFTransform, MF_E_INVALIDMEDIATYPE, MF_E_NO_MORE_TYPES, + MF_E_TRANSFORM_TYPE_NOT_SET, MF_EVENT_FLAG_NONE, MF_EVENT_TYPE, + MF_MT_ALL_SAMPLES_INDEPENDENT, MF_MT_AVG_BITRATE, MF_MT_FRAME_RATE, MF_MT_FRAME_SIZE, + MF_MT_INTERLACE_MODE, MF_MT_MAJOR_TYPE, MF_MT_PIXEL_ASPECT_RATIO, MF_MT_SUBTYPE, + MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS, MF_TRANSFORM_ASYNC_UNLOCK, + MFCreateDXGIDeviceManager, MFCreateDXGISurfaceBuffer, MFCreateMediaType, + MFCreateSample, MFMediaType_Video, MFT_ENUM_FLAG, MFT_ENUM_FLAG_HARDWARE, + MFT_ENUM_FLAG_TRANSCODE_ONLY, MFT_MESSAGE_COMMAND_FLUSH, + MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, MFT_MESSAGE_NOTIFY_END_OF_STREAM, + MFT_MESSAGE_NOTIFY_END_STREAMING, MFT_MESSAGE_NOTIFY_START_OF_STREAM, + MFT_MESSAGE_SET_D3D_MANAGER, MFT_OUTPUT_DATA_BUFFER, MFT_SET_TYPE_TEST_ONLY, + MFVideoFormat_HEVC, MFVideoFormat_NV12, MFVideoInterlace_Progressive, + }, + }, + core::{Error, Interface}, +}; + +const MAX_CONSECUTIVE_EMPTY_SAMPLES: u8 = 20; + +pub struct HevcEncoder { + _d3d_device: ID3D11Device, + _media_device_manager: IMFDXGIDeviceManager, + _device_manager_reset_token: u32, + + video_processor: VideoProcessor, + + transform: IMFTransform, + event_generator: IMFMediaEventGenerator, + input_stream_id: u32, + output_stream_id: u32, + output_type: IMFMediaType, + bitrate: u32, +} + +#[derive(Clone, Debug, thiserror::Error)] +pub enum NewHevcEncoderError { + #[error("NoVideoEncoderDevice")] + NoVideoEncoderDevice, + #[error("EncoderTransform: {0}")] + EncoderTransform(windows::core::Error), + #[error("VideoProcessor: {0}")] + VideoProcessor(NewVideoProcessorError), + #[error("DeviceManager: {0}")] + DeviceManager(windows::core::Error), + #[error("EventGenerator: {0}")] + EventGenerator(windows::core::Error), + #[error("ConfigureStreams: {0}")] + ConfigureStreams(windows::core::Error), + #[error("OutputType: {0}")] + OutputType(windows::core::Error), + #[error("InputType: {0}")] + InputType(windows::core::Error), +} + +unsafe impl Send for HevcEncoder {} + +impl HevcEncoder { + #[allow(clippy::too_many_arguments)] + fn new_with_scaled_output_with_flags( + d3d_device: &ID3D11Device, + format: DXGI_FORMAT, + input_resolution: SizeInt32, + output_resolution: SizeInt32, + frame_rate: u32, + bitrate_multipler: f32, + flags: MFT_ENUM_FLAG, + enable_hardware_transforms: bool, + ) -> Result { + let bitrate = calculate_bitrate( + output_resolution.Width as u32, + output_resolution.Height as u32, + frame_rate, + bitrate_multipler, + ); + + let transform = + EncoderDevice::enumerate_with_flags(MFMediaType_Video, MFVideoFormat_HEVC, flags) + .map_err(|_| NewHevcEncoderError::NoVideoEncoderDevice)? + .first() + .cloned() + .ok_or(NewHevcEncoderError::NoVideoEncoderDevice)? + .create_transform() + .map_err(NewHevcEncoderError::EncoderTransform)?; + + let video_processor = VideoProcessor::new( + d3d_device.clone(), + format, + input_resolution, + DXGI_FORMAT_NV12, + output_resolution, + frame_rate, + ) + .map_err(NewHevcEncoderError::VideoProcessor)?; + + let mut device_manager_reset_token: u32 = 0; + let media_device_manager = { + let mut media_device_manager = None; + unsafe { + MFCreateDXGIDeviceManager( + &mut device_manager_reset_token, + &mut media_device_manager, + ) + .map_err(NewHevcEncoderError::DeviceManager)? + }; + media_device_manager.expect("Device manager unexpectedly None") + }; + unsafe { + media_device_manager + .ResetDevice(d3d_device, device_manager_reset_token) + .map_err(NewHevcEncoderError::DeviceManager)? + }; + + let event_generator: IMFMediaEventGenerator = transform + .cast() + .map_err(NewHevcEncoderError::EventGenerator)?; + let attributes = unsafe { + transform + .GetAttributes() + .map_err(NewHevcEncoderError::EventGenerator)? + }; + unsafe { + attributes + .SetUINT32(&MF_TRANSFORM_ASYNC_UNLOCK, 1) + .map_err(NewHevcEncoderError::EventGenerator)?; + attributes + .SetUINT32( + &MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS, + enable_hardware_transforms as u32, + ) + .map_err(NewHevcEncoderError::EventGenerator)?; + }; + + let mut number_of_input_streams = 0; + let mut number_of_output_streams = 0; + unsafe { + transform + .GetStreamCount(&mut number_of_input_streams, &mut number_of_output_streams) + .map_err(NewHevcEncoderError::EventGenerator)? + }; + let (input_stream_ids, output_stream_ids) = { + let mut input_stream_ids = vec![0u32; number_of_input_streams as usize]; + let mut output_stream_ids = vec![0u32; number_of_output_streams as usize]; + let result = + unsafe { transform.GetStreamIDs(&mut input_stream_ids, &mut output_stream_ids) }; + match result { + Ok(_) => {} + Err(error) => { + if error.code() == E_NOTIMPL { + for i in 0..number_of_input_streams { + input_stream_ids[i as usize] = i; + } + for i in 0..number_of_output_streams { + output_stream_ids[i as usize] = i; + } + } else { + return Err(NewHevcEncoderError::ConfigureStreams(error)); + } + } + } + (input_stream_ids, output_stream_ids) + }; + let input_stream_id = input_stream_ids[0]; + let output_stream_id = output_stream_ids[0]; + + unsafe { + let temp = media_device_manager.clone(); + transform + .ProcessMessage( + MFT_MESSAGE_SET_D3D_MANAGER, + std::mem::transmute::(temp), + ) + .map_err(NewHevcEncoderError::EncoderTransform)?; + }; + + let output_type = (|| unsafe { + let output_type = MFCreateMediaType()?; + let attributes: IMFAttributes = output_type.cast()?; + output_type.SetGUID(&MF_MT_MAJOR_TYPE, &MFMediaType_Video)?; + output_type.SetGUID(&MF_MT_SUBTYPE, &MFVideoFormat_HEVC)?; + output_type.SetUINT32(&MF_MT_AVG_BITRATE, bitrate)?; + MFSetAttributeSize( + &attributes, + &MF_MT_FRAME_SIZE, + output_resolution.Width as u32, + output_resolution.Height as u32, + )?; + MFSetAttributeRatio(&attributes, &MF_MT_FRAME_RATE, frame_rate, 1)?; + MFSetAttributeRatio(&attributes, &MF_MT_PIXEL_ASPECT_RATIO, 1, 1)?; + output_type.SetUINT32(&MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive.0 as u32)?; + output_type.SetUINT32(&MF_MT_ALL_SAMPLES_INDEPENDENT, 1)?; + transform.SetOutputType(output_stream_id, &output_type, 0)?; + Ok(output_type) + })() + .map_err(NewHevcEncoderError::OutputType)?; + + let input_type: Option = (|| unsafe { + let mut count = 0; + loop { + let result = transform.GetInputAvailableType(input_stream_id, count); + if let Err(error) = &result + && error.code() == MF_E_NO_MORE_TYPES + { + break Ok(None); + } + + let input_type = result?; + let attributes: IMFAttributes = input_type.cast()?; + input_type.SetGUID(&MF_MT_MAJOR_TYPE, &MFMediaType_Video)?; + input_type.SetGUID(&MF_MT_SUBTYPE, &MFVideoFormat_NV12)?; + MFSetAttributeSize( + &attributes, + &MF_MT_FRAME_SIZE, + output_resolution.Width as u32, + output_resolution.Height as u32, + )?; + MFSetAttributeRatio(&attributes, &MF_MT_FRAME_RATE, frame_rate, 1)?; + let result = transform.SetInputType( + input_stream_id, + &input_type, + MFT_SET_TYPE_TEST_ONLY.0 as u32, + ); + if let Err(error) = &result + && error.code() == MF_E_INVALIDMEDIATYPE + { + count += 1; + continue; + } + result?; + break Ok(Some(input_type)); + } + })() + .map_err(NewHevcEncoderError::InputType)?; + if let Some(input_type) = input_type { + unsafe { transform.SetInputType(input_stream_id, &input_type, 0) } + .map_err(NewHevcEncoderError::InputType)?; + } else { + return Err(NewHevcEncoderError::InputType(Error::new( + MF_E_TRANSFORM_TYPE_NOT_SET, + "No suitable input type found! Try a different set of encoding settings.", + ))); + } + + Ok(Self { + _d3d_device: d3d_device.clone(), + _media_device_manager: media_device_manager, + _device_manager_reset_token: device_manager_reset_token, + + video_processor, + + transform, + event_generator, + input_stream_id, + output_stream_id, + bitrate, + + output_type, + }) + } + + pub fn new_with_scaled_output( + d3d_device: &ID3D11Device, + format: DXGI_FORMAT, + input_resolution: SizeInt32, + output_resolution: SizeInt32, + frame_rate: u32, + bitrate_multipler: f32, + ) -> Result { + Self::new_with_scaled_output_with_flags( + d3d_device, + format, + input_resolution, + output_resolution, + frame_rate, + bitrate_multipler, + MFT_ENUM_FLAG_HARDWARE | MFT_ENUM_FLAG_TRANSCODE_ONLY, + true, + ) + } + + pub fn new_with_scaled_output_software( + d3d_device: &ID3D11Device, + format: DXGI_FORMAT, + input_resolution: SizeInt32, + output_resolution: SizeInt32, + frame_rate: u32, + bitrate_multipler: f32, + ) -> Result { + Self::new_with_scaled_output_with_flags( + d3d_device, + format, + input_resolution, + output_resolution, + frame_rate, + bitrate_multipler, + MFT_ENUM_FLAG_TRANSCODE_ONLY, + false, + ) + } + + pub fn new( + d3d_device: &ID3D11Device, + format: DXGI_FORMAT, + resolution: SizeInt32, + frame_rate: u32, + bitrate_multipler: f32, + ) -> Result { + Self::new_with_scaled_output( + d3d_device, + format, + resolution, + resolution, + frame_rate, + bitrate_multipler, + ) + } + + pub fn new_software( + d3d_device: &ID3D11Device, + format: DXGI_FORMAT, + resolution: SizeInt32, + frame_rate: u32, + bitrate_multipler: f32, + ) -> Result { + Self::new_with_scaled_output_software( + d3d_device, + format, + resolution, + resolution, + frame_rate, + bitrate_multipler, + ) + } + + pub fn bitrate(&self) -> u32 { + self.bitrate + } + + pub fn output_type(&self) -> &IMFMediaType { + &self.output_type + } + + pub fn run( + &mut self, + should_stop: Arc, + mut get_frame: impl FnMut() -> windows::core::Result>, + mut on_sample: impl FnMut(IMFSample) -> windows::core::Result<()>, + ) -> windows::core::Result<()> { + unsafe { + self.transform + .ProcessMessage(MFT_MESSAGE_COMMAND_FLUSH, 0)?; + self.transform + .ProcessMessage(MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, 0)?; + self.transform + .ProcessMessage(MFT_MESSAGE_NOTIFY_START_OF_STREAM, 0)?; + + let mut consecutive_empty_samples = 0; + let mut should_exit = false; + while !should_exit { + let event = self.event_generator.GetEvent(MF_EVENT_FLAG_NONE)?; + + let event_type = MF_EVENT_TYPE(event.GetType()? as i32); + match event_type { + MediaFoundation::METransformNeedInput => { + should_exit = true; + if !should_stop.load(Ordering::SeqCst) + && let Some((texture, timestamp)) = get_frame()? + { + self.video_processor.process_texture(&texture)?; + let input_buffer = { + MFCreateDXGISurfaceBuffer( + &ID3D11Texture2D::IID, + self.video_processor.output_texture(), + 0, + false, + )? + }; + let mf_sample = MFCreateSample()?; + mf_sample.AddBuffer(&input_buffer)?; + mf_sample.SetSampleTime(timestamp.Duration)?; + self.transform + .ProcessInput(self.input_stream_id, &mf_sample, 0)?; + should_exit = false; + } + } + MediaFoundation::METransformHaveOutput => { + let mut status = 0; + let output_buffer = MFT_OUTPUT_DATA_BUFFER { + dwStreamID: self.output_stream_id, + ..Default::default() + }; + + let mut output_buffers = [output_buffer]; + self.transform + .ProcessOutput(0, &mut output_buffers, &mut status)?; + + if let Some(sample) = output_buffers[0].pSample.take() { + consecutive_empty_samples = 0; + on_sample(sample)?; + } else { + consecutive_empty_samples += 1; + if consecutive_empty_samples > MAX_CONSECUTIVE_EMPTY_SAMPLES { + return Err(windows::core::Error::new( + windows::core::HRESULT(-1), + "Too many consecutive empty samples", + )); + } + } + } + _ => { + tracing::warn!("Ignoring unknown media event type: {}", event_type.0); + } + } + } + + self.transform + .ProcessMessage(MFT_MESSAGE_NOTIFY_END_OF_STREAM, 0)?; + self.transform + .ProcessMessage(MFT_MESSAGE_NOTIFY_END_STREAMING, 0)?; + self.transform + .ProcessMessage(MFT_MESSAGE_COMMAND_FLUSH, 0)?; + } + + Ok(()) + } +} + +fn calculate_bitrate(width: u32, height: u32, fps: u32, multiplier: f32) -> u32 { + let frame_rate_factor = (fps as f32 - 30.0).max(0.0) / 2.0 + 30.0; + (width as f32 * height as f32 * frame_rate_factor * multiplier * 0.6) as u32 +} diff --git a/crates/enc-mediafoundation/src/video/mod.rs b/crates/enc-mediafoundation/src/video/mod.rs index 273711713b..b6376900bd 100644 --- a/crates/enc-mediafoundation/src/video/mod.rs +++ b/crates/enc-mediafoundation/src/video/mod.rs @@ -1,5 +1,7 @@ mod h264; +mod hevc; mod video_processor; pub use h264::*; +pub use hevc::*; pub use video_processor::*; diff --git a/crates/frame-converter/Cargo.toml b/crates/frame-converter/Cargo.toml index a69a81dbba..39b4e7ce78 100644 --- a/crates/frame-converter/Cargo.toml +++ b/crates/frame-converter/Cargo.toml @@ -24,7 +24,9 @@ windows = { workspace = true, features = [ "Win32_Foundation", "Win32_Graphics_Direct3D", "Win32_Graphics_Direct3D11", + "Win32_Graphics_Dxgi", "Win32_Graphics_Dxgi_Common", "Win32_Media_MediaFoundation", + "Win32_Security", ] } diff --git a/crates/frame-converter/src/d3d11.rs b/crates/frame-converter/src/d3d11.rs index 60eaaefd45..fccc462ab7 100644 --- a/crates/frame-converter/src/d3d11.rs +++ b/crates/frame-converter/src/d3d11.rs @@ -4,17 +4,21 @@ use parking_lot::Mutex; use std::{ mem::ManuallyDrop, ptr, - sync::atomic::{AtomicBool, AtomicU64, Ordering}, + sync::{ + OnceLock, + atomic::{AtomicBool, AtomicU64, Ordering}, + }, + time::Instant, }; use windows::{ Win32::{ - Foundation::HMODULE, + Foundation::{CloseHandle, HANDLE, HMODULE}, Graphics::{ - Direct3D::D3D_DRIVER_TYPE_HARDWARE, Direct3D11::{ D3D11_BIND_RENDER_TARGET, D3D11_CPU_ACCESS_READ, D3D11_CPU_ACCESS_WRITE, D3D11_CREATE_DEVICE_VIDEO_SUPPORT, D3D11_MAP_READ, D3D11_MAP_WRITE, - D3D11_MAPPED_SUBRESOURCE, D3D11_SDK_VERSION, D3D11_TEXTURE2D_DESC, + D3D11_MAPPED_SUBRESOURCE, D3D11_RESOURCE_MISC_SHARED, + D3D11_RESOURCE_MISC_SHARED_NTHANDLE, D3D11_SDK_VERSION, D3D11_TEXTURE2D_DESC, D3D11_USAGE_DEFAULT, D3D11_USAGE_STAGING, D3D11_VIDEO_PROCESSOR_CONTENT_DESC, D3D11_VIDEO_PROCESSOR_INPUT_VIEW_DESC, D3D11_VIDEO_PROCESSOR_OUTPUT_VIEW_DESC, D3D11_VIDEO_PROCESSOR_STREAM, D3D11_VPIV_DIMENSION_TEXTURE2D, @@ -24,8 +28,12 @@ use windows::{ ID3D11VideoProcessorInputView, ID3D11VideoProcessorOutputView, }, Dxgi::{ - Common::{DXGI_FORMAT, DXGI_FORMAT_NV12, DXGI_FORMAT_YUY2}, - IDXGIAdapter, IDXGIDevice, + Common::{ + DXGI_FORMAT, DXGI_FORMAT_B8G8R8A8_UNORM, DXGI_FORMAT_NV12, DXGI_FORMAT_P010, + DXGI_FORMAT_R8G8B8A8_UNORM, DXGI_FORMAT_YUY2, + }, + CreateDXGIFactory1, DXGI_SHARED_RESOURCE_READ, DXGI_SHARED_RESOURCE_WRITE, + IDXGIAdapter, IDXGIDevice, IDXGIFactory1, IDXGIResource1, }, }, }, @@ -64,6 +72,26 @@ pub struct GpuInfo { pub device_id: u32, pub description: String, pub dedicated_video_memory: u64, + pub adapter_index: u32, + pub is_software_adapter: bool, +} + +impl GpuInfo { + pub fn is_basic_render_driver(&self) -> bool { + self.vendor == GpuVendor::Microsoft + && (self.description.contains("Basic Render Driver") + || self.description.contains("Microsoft Basic") + || self.dedicated_video_memory == 0) + } + + pub fn is_warp(&self) -> bool { + self.description.contains("Microsoft Basic Render Driver") + || self.description.contains("WARP") + } + + pub fn supports_hardware_encoding(&self) -> bool { + !self.is_software_adapter && !self.is_basic_render_driver() + } } impl GpuInfo { @@ -80,6 +108,161 @@ impl GpuInfo { } } +static DETECTED_GPU: OnceLock> = OnceLock::new(); +static ALL_GPUS: OnceLock> = OnceLock::new(); + +pub fn detect_primary_gpu() -> Option<&'static GpuInfo> { + DETECTED_GPU + .get_or_init(|| { + let all_gpus = enumerate_all_gpus(); + let result = select_best_gpu(&all_gpus); + if let Some(ref info) = result { + tracing::debug!( + "Selected primary GPU: {} (Vendor: {}, VendorID: 0x{:04X}, VRAM: {} MB, Adapter: {}, SoftwareRenderer: {})", + info.description, + info.vendor_name(), + info.vendor_id, + info.dedicated_video_memory / (1024 * 1024), + info.adapter_index, + info.is_software_adapter + ); + + if info.is_basic_render_driver() { + tracing::warn!( + "Detected Microsoft Basic Render Driver - hardware encoding will be disabled. \ + This may indicate missing GPU drivers or a remote desktop session." + ); + } + } else { + tracing::debug!("No GPU detected via DXGI, using default encoder order"); + } + result + }) + .as_ref() +} + +pub fn get_all_gpus() -> &'static Vec { + ALL_GPUS.get_or_init(enumerate_all_gpus) +} + +fn enumerate_all_gpus() -> Vec { + let mut gpus = Vec::new(); + + unsafe { + let factory: IDXGIFactory1 = match CreateDXGIFactory1() { + Ok(f) => f, + Err(e) => { + tracing::warn!("Failed to create DXGI factory: {e:?}"); + return gpus; + } + }; + + let mut adapter_index = 0u32; + loop { + let adapter: IDXGIAdapter = match factory.EnumAdapters(adapter_index) { + Ok(a) => a, + Err(_) => break, + }; + + if let Ok(desc) = adapter.GetDesc() { + let description = String::from_utf16_lossy( + &desc + .Description + .iter() + .take_while(|&&c| c != 0) + .copied() + .collect::>(), + ); + + let is_software = desc.VendorId == 0x1414 + && (description.contains("Basic Render") + || description.contains("WARP") + || description.contains("Microsoft Basic")); + + let gpu_info = GpuInfo { + vendor: GpuVendor::from_id(desc.VendorId), + vendor_id: desc.VendorId, + device_id: desc.DeviceId, + description: description.clone(), + dedicated_video_memory: desc.DedicatedVideoMemory as u64, + adapter_index, + is_software_adapter: is_software, + }; + + tracing::debug!( + "Found GPU adapter {}: {} (Vendor: 0x{:04X}, VRAM: {} MB, Software: {})", + adapter_index, + description, + desc.VendorId, + desc.DedicatedVideoMemory / (1024 * 1024), + is_software + ); + + gpus.push(gpu_info); + } + + adapter_index += 1; + } + } + + if gpus.is_empty() { + tracing::warn!("No GPU adapters found via DXGI enumeration"); + } else { + tracing::info!("Enumerated {} GPU adapter(s)", gpus.len()); + } + + gpus +} + +fn select_best_gpu(gpus: &[GpuInfo]) -> Option { + if gpus.is_empty() { + return None; + } + + let hardware_gpus: Vec<&GpuInfo> = gpus.iter().filter(|g| !g.is_software_adapter).collect(); + + if hardware_gpus.is_empty() { + tracing::warn!("No hardware GPUs found, falling back to software adapter"); + return gpus.first().cloned(); + } + + let discrete_gpus: Vec<&GpuInfo> = hardware_gpus + .iter() + .filter(|g| { + matches!( + g.vendor, + GpuVendor::Nvidia | GpuVendor::Amd | GpuVendor::Qualcomm | GpuVendor::Arm + ) + }) + .copied() + .collect(); + + if !discrete_gpus.is_empty() { + let best = discrete_gpus + .iter() + .max_by_key(|g| g.dedicated_video_memory) + .unwrap(); + + if discrete_gpus.len() > 1 || hardware_gpus.len() > 1 { + tracing::info!( + "Multi-GPU system detected ({} GPUs). Selected {} with {} MB VRAM for encoding.", + gpus.len(), + best.description, + best.dedicated_video_memory / (1024 * 1024) + ); + } + + return Some((*best).clone()); + } + + let best = hardware_gpus + .iter() + .max_by_key(|g| g.dedicated_video_memory) + .unwrap(); + + Some((*best).clone()) +} + struct D3D11Resources { #[allow(dead_code)] device: ID3D11Device, @@ -90,24 +273,24 @@ struct D3D11Resources { enumerator: ID3D11VideoProcessorEnumerator, input_texture: ID3D11Texture2D, output_texture: ID3D11Texture2D, + input_shared_handle: Option, + output_shared_handle: Option, staging_input: ID3D11Texture2D, staging_output: ID3D11Texture2D, } pub struct D3D11Converter { resources: Mutex, - #[allow(dead_code)] input_format: Pixel, output_format: Pixel, - #[allow(dead_code)] input_width: u32, - #[allow(dead_code)] input_height: u32, output_width: u32, output_height: u32, gpu_info: GpuInfo, conversion_count: AtomicU64, verified_gpu_usage: AtomicBool, + total_conversion_time_ns: AtomicU64, } fn get_gpu_info(device: &ID3D11Device) -> Result { @@ -133,28 +316,115 @@ fn get_gpu_info(device: &ID3D11Device) -> Result { .collect::>(), ); + let is_software = desc.VendorId == 0x1414 + && (description.contains("Basic Render") + || description.contains("WARP") + || description.contains("Microsoft Basic")); + Ok(GpuInfo { vendor: GpuVendor::from_id(desc.VendorId), vendor_id: desc.VendorId, device_id: desc.DeviceId, description, dedicated_video_memory: desc.DedicatedVideoMemory as u64, + adapter_index: 0, + is_software_adapter: is_software, }) } } impl D3D11Converter { - pub fn new(config: ConversionConfig) -> Result { - let input_dxgi = pixel_to_dxgi(config.input_format)?; - let output_dxgi = pixel_to_dxgi(config.output_format)?; + fn create_device_on_best_adapter() -> Result<(ID3D11Device, ID3D11DeviceContext), ConvertError> + { + unsafe { + let factory: IDXGIFactory1 = CreateDXGIFactory1().map_err(|e| { + ConvertError::HardwareUnavailable(format!("Failed to create DXGI factory: {e:?}")) + })?; + + let mut best_adapter: Option<(IDXGIAdapter, String, u64)> = None; + let mut adapter_index = 0u32; + + loop { + let adapter: IDXGIAdapter = match factory.EnumAdapters(adapter_index) { + Ok(a) => a, + Err(_) => break, + }; + + if let Ok(desc) = adapter.GetDesc() { + let description = String::from_utf16_lossy( + &desc + .Description + .iter() + .take_while(|&&c| c != 0) + .copied() + .collect::>(), + ); + + let is_software = desc.VendorId == 0x1414 + && (description.contains("Basic Render") + || description.contains("WARP") + || description.contains("Microsoft Basic")); + + if !is_software { + let vendor = GpuVendor::from_id(desc.VendorId); + let is_discrete = matches!( + vendor, + GpuVendor::Nvidia + | GpuVendor::Amd + | GpuVendor::Qualcomm + | GpuVendor::Arm + ); + + let vram = desc.DedicatedVideoMemory as u64; + + let should_use = match &best_adapter { + None => true, + Some((_, _, best_vram)) => { + if is_discrete { + vram > *best_vram + } else { + false + } + } + }; + + if should_use { + tracing::debug!( + "D3D11: Candidate adapter {}: {} (Vendor: 0x{:04X}, VRAM: {} MB, Discrete: {})", + adapter_index, + description, + desc.VendorId, + vram / (1024 * 1024), + is_discrete + ); + best_adapter = Some((adapter, description, vram)); + } + } else { + tracing::debug!( + "D3D11: Skipping software adapter {}: {}", + adapter_index, + description + ); + } + } + + adapter_index += 1; + } + + let (adapter, adapter_name, _) = best_adapter.ok_or_else(|| { + ConvertError::HardwareUnavailable( + "No suitable hardware GPU adapter found".to_string(), + ) + })?; + + tracing::info!("D3D11: Creating device on adapter: {}", adapter_name); - let (device, context) = unsafe { let mut device = None; let mut context = None; D3D11CreateDevice( - None, - D3D_DRIVER_TYPE_HARDWARE, + Some(&adapter), + windows::Win32::Graphics::Direct3D::D3D_DRIVER_TYPE_UNKNOWN, HMODULE::default(), D3D11_CREATE_DEVICE_VIDEO_SUPPORT, None, @@ -165,7 +435,7 @@ impl D3D11Converter { ) .map_err(|e| { ConvertError::HardwareUnavailable(format!( - "D3D11CreateDevice failed (no hardware GPU available?): {e:?}" + "D3D11CreateDevice failed on {adapter_name}: {e:?}" )) })?; @@ -176,8 +446,15 @@ impl D3D11Converter { ConvertError::HardwareUnavailable("D3D11 context was null".to_string()) })?; - (device, context) - }; + Ok((device, context)) + } + } + + pub fn new(config: ConversionConfig) -> Result { + let input_dxgi = pixel_to_dxgi(config.input_format)?; + let output_dxgi = pixel_to_dxgi(config.output_format)?; + + let (device, context) = Self::create_device_on_best_adapter()?; let gpu_info = get_gpu_info(&device)?; @@ -237,24 +514,20 @@ impl D3D11Converter { })? }; - let input_texture = create_texture( + let (input_texture, input_shared_handle) = create_shared_texture( &device, config.input_width, config.input_height, input_dxgi, - D3D11_USAGE_DEFAULT, D3D11_BIND_RENDER_TARGET.0 as u32, - 0, )?; - let output_texture = create_texture( + let (output_texture, output_shared_handle) = create_shared_texture( &device, config.output_width, config.output_height, output_dxgi, - D3D11_USAGE_DEFAULT, D3D11_BIND_RENDER_TARGET.0 as u32, - 0, )?; let staging_input = create_texture( @@ -277,6 +550,16 @@ impl D3D11Converter { D3D11_CPU_ACCESS_READ.0 as u32, )?; + if input_shared_handle.is_some() && output_shared_handle.is_some() { + tracing::info!("D3D11 converter created with shared texture handles enabled"); + } else { + tracing::warn!( + "D3D11 converter created without shared handles (input: {}, output: {})", + input_shared_handle.is_some(), + output_shared_handle.is_some() + ); + } + let resources = D3D11Resources { device, context, @@ -286,6 +569,8 @@ impl D3D11Converter { enumerator, input_texture, output_texture, + input_shared_handle, + output_shared_handle, staging_input, staging_output, }; @@ -312,21 +597,80 @@ impl D3D11Converter { gpu_info, conversion_count: AtomicU64::new(0), verified_gpu_usage: AtomicBool::new(false), + total_conversion_time_ns: AtomicU64::new(0), }) } pub fn gpu_info(&self) -> &GpuInfo { &self.gpu_info } + + pub fn input_shared_handle(&self) -> Option { + self.resources.lock().input_shared_handle + } + + pub fn output_shared_handle(&self) -> Option { + self.resources.lock().output_shared_handle + } + + pub fn output_texture(&self) -> ID3D11Texture2D { + self.resources.lock().output_texture.clone() + } + + pub fn has_shared_handles(&self) -> bool { + let resources = self.resources.lock(); + resources.input_shared_handle.is_some() && resources.output_shared_handle.is_some() + } + + pub fn average_conversion_time_ms(&self) -> Option { + let count = self.conversion_count.load(Ordering::Relaxed); + if count == 0 { + return None; + } + let total_ns = self.total_conversion_time_ns.load(Ordering::Relaxed); + Some((total_ns as f64 / count as f64) / 1_000_000.0) + } + + pub fn format_info(&self) -> (Pixel, Pixel, u32, u32, u32, u32) { + ( + self.input_format, + self.output_format, + self.input_width, + self.input_height, + self.output_width, + self.output_height, + ) + } +} + +pub fn supported_input_formats() -> &'static [Pixel] { + &[ + Pixel::NV12, + Pixel::YUYV422, + Pixel::BGRA, + Pixel::RGBA, + Pixel::P010LE, + ] +} + +pub fn is_format_supported(pixel: Pixel) -> bool { + pixel_to_dxgi(pixel).is_ok() } impl FrameConverter for D3D11Converter { fn convert(&self, input: frame::Video) -> Result { + let start = Instant::now(); let count = self.conversion_count.fetch_add(1, Ordering::Relaxed); if count == 0 { tracing::info!( - "D3D11 converter first frame: converting on GPU {} ({})", + "D3D11 converter first frame: {:?} {}x{} -> {:?} {}x{} on GPU {} ({})", + self.input_format, + self.input_width, + self.input_height, + self.output_format, + self.output_width, + self.output_height, self.gpu_info.description, self.gpu_info.vendor_name() ); @@ -465,6 +809,25 @@ impl FrameConverter for D3D11Converter { resources.context.Unmap(&resources.staging_output, 0); output.set_pts(pts); + + let elapsed_ns = start.elapsed().as_nanos() as u64; + self.total_conversion_time_ns + .fetch_add(elapsed_ns, Ordering::Relaxed); + + let frame_count = count + 1; + if frame_count.is_multiple_of(300) { + let total_ns = self.total_conversion_time_ns.load(Ordering::Relaxed); + let avg_ms = (total_ns as f64 / frame_count as f64) / 1_000_000.0; + tracing::debug!( + "D3D11 converter: {:.2}ms avg over {} frames ({:?} -> {:?}) on {}", + avg_ms, + frame_count, + self.input_format, + self.output_format, + self.gpu_info.description + ); + } + Ok(output) } } @@ -490,10 +853,25 @@ fn pixel_to_dxgi(pixel: Pixel) -> Result { match pixel { Pixel::NV12 => Ok(DXGI_FORMAT_NV12), Pixel::YUYV422 => Ok(DXGI_FORMAT_YUY2), + Pixel::BGRA => Ok(DXGI_FORMAT_B8G8R8A8_UNORM), + Pixel::RGBA => Ok(DXGI_FORMAT_R8G8B8A8_UNORM), + Pixel::P010LE => Ok(DXGI_FORMAT_P010), _ => Err(ConvertError::UnsupportedFormat(pixel, Pixel::NV12)), } } +#[allow(dead_code)] +fn dxgi_format_name(format: DXGI_FORMAT) -> &'static str { + match format { + DXGI_FORMAT_NV12 => "NV12", + DXGI_FORMAT_YUY2 => "YUY2", + DXGI_FORMAT_B8G8R8A8_UNORM => "BGRA", + DXGI_FORMAT_R8G8B8A8_UNORM => "RGBA", + DXGI_FORMAT_P010 => "P010", + _ => "Unknown", + } +} + fn create_texture( device: &ID3D11Device, width: u32, @@ -532,8 +910,84 @@ fn create_texture( } } +fn create_shared_texture( + device: &ID3D11Device, + width: u32, + height: u32, + format: DXGI_FORMAT, + bind_flags: u32, +) -> Result<(ID3D11Texture2D, Option), ConvertError> { + let desc = D3D11_TEXTURE2D_DESC { + Width: width, + Height: height, + MipLevels: 1, + ArraySize: 1, + Format: format, + SampleDesc: windows::Win32::Graphics::Dxgi::Common::DXGI_SAMPLE_DESC { + Count: 1, + Quality: 0, + }, + Usage: D3D11_USAGE_DEFAULT, + BindFlags: bind_flags, + CPUAccessFlags: 0, + MiscFlags: (D3D11_RESOURCE_MISC_SHARED.0 | D3D11_RESOURCE_MISC_SHARED_NTHANDLE.0) as u32, + }; + + let texture = unsafe { + let mut texture: Option = None; + device + .CreateTexture2D(&desc, None, Some(&mut texture)) + .map_err(|e| { + ConvertError::HardwareUnavailable(format!( + "CreateTexture2D with shared handle failed: {e:?}" + )) + })?; + texture.ok_or_else(|| { + ConvertError::HardwareUnavailable("CreateTexture2D returned null".to_string()) + })? + }; + + let shared_handle = extract_shared_handle(&texture); + + Ok((texture, shared_handle)) +} + +fn extract_shared_handle(texture: &ID3D11Texture2D) -> Option { + unsafe { + let dxgi_resource: Result = texture.cast(); + match dxgi_resource { + Ok(resource) => { + let result = resource.CreateSharedHandle( + None, + DXGI_SHARED_RESOURCE_READ.0 | DXGI_SHARED_RESOURCE_WRITE.0, + None, + ); + match result { + Ok(handle) if !handle.is_invalid() => { + tracing::debug!("Created shared handle for texture: {:?}", handle); + Some(handle) + } + Ok(_) => { + tracing::warn!("CreateSharedHandle returned invalid handle"); + None + } + Err(e) => { + tracing::warn!("CreateSharedHandle failed: {e:?}"); + None + } + } + } + Err(e) => { + tracing::warn!("Failed to cast texture to IDXGIResource1: {e:?}"); + None + } + } + } +} + unsafe fn copy_frame_to_mapped(frame: &frame::Video, dst: *mut u8, dst_stride: usize) { let height = frame.height() as usize; + let width = frame.width() as usize; let format = frame.format(); match format { @@ -543,7 +997,7 @@ unsafe fn copy_frame_to_mapped(frame: &frame::Video, dst: *mut u8, dst_stride: u ptr::copy_nonoverlapping( frame.data(0).as_ptr().add(y * frame.stride(0)), dst.add(y * dst_stride), - frame.width() as usize, + width, ); } } @@ -553,13 +1007,25 @@ unsafe fn copy_frame_to_mapped(frame: &frame::Video, dst: *mut u8, dst_stride: u ptr::copy_nonoverlapping( frame.data(1).as_ptr().add(y * frame.stride(1)), dst.add(uv_offset + y * dst_stride), - frame.width() as usize, + width, ); } } } Pixel::YUYV422 => { - let row_bytes = frame.width() as usize * 2; + let row_bytes = width * 2; + for y in 0..height { + unsafe { + ptr::copy_nonoverlapping( + frame.data(0).as_ptr().add(y * frame.stride(0)), + dst.add(y * dst_stride), + row_bytes, + ); + } + } + } + Pixel::BGRA | Pixel::RGBA => { + let row_bytes = width * 4; for y in 0..height { unsafe { ptr::copy_nonoverlapping( @@ -570,12 +1036,35 @@ unsafe fn copy_frame_to_mapped(frame: &frame::Video, dst: *mut u8, dst_stride: u } } } + Pixel::P010LE => { + let row_bytes = width * 2; + for y in 0..height { + unsafe { + ptr::copy_nonoverlapping( + frame.data(0).as_ptr().add(y * frame.stride(0)), + dst.add(y * dst_stride), + row_bytes, + ); + } + } + let uv_offset = height * dst_stride; + for y in 0..height / 2 { + unsafe { + ptr::copy_nonoverlapping( + frame.data(1).as_ptr().add(y * frame.stride(1)), + dst.add(uv_offset + y * dst_stride), + row_bytes, + ); + } + } + } _ => {} } } unsafe fn copy_mapped_to_frame(src: *const u8, src_stride: usize, frame: &mut frame::Video) { let height = frame.height() as usize; + let width = frame.width() as usize; let format = frame.format(); match format { @@ -585,7 +1074,7 @@ unsafe fn copy_mapped_to_frame(src: *const u8, src_stride: usize, frame: &mut fr ptr::copy_nonoverlapping( src.add(y * src_stride), frame.data_mut(0).as_mut_ptr().add(y * frame.stride(0)), - frame.width() as usize, + width, ); } } @@ -595,14 +1084,13 @@ unsafe fn copy_mapped_to_frame(src: *const u8, src_stride: usize, frame: &mut fr ptr::copy_nonoverlapping( src.add(uv_offset + y * src_stride), frame.data_mut(1).as_mut_ptr().add(y * frame.stride(1)), - frame.width() as usize, + width, ); } } } Pixel::YUYV422 => { - let bytes_per_pixel = 2; - let row_bytes = frame.width() as usize * bytes_per_pixel; + let row_bytes = width * 2; for y in 0..height { unsafe { ptr::copy_nonoverlapping( @@ -613,9 +1101,62 @@ unsafe fn copy_mapped_to_frame(src: *const u8, src_stride: usize, frame: &mut fr } } } + Pixel::BGRA | Pixel::RGBA => { + let row_bytes = width * 4; + for y in 0..height { + unsafe { + ptr::copy_nonoverlapping( + src.add(y * src_stride), + frame.data_mut(0).as_mut_ptr().add(y * frame.stride(0)), + row_bytes, + ); + } + } + } + Pixel::P010LE => { + let row_bytes = width * 2; + for y in 0..height { + unsafe { + ptr::copy_nonoverlapping( + src.add(y * src_stride), + frame.data_mut(0).as_mut_ptr().add(y * frame.stride(0)), + row_bytes, + ); + } + } + let uv_offset = height * src_stride; + for y in 0..height / 2 { + unsafe { + ptr::copy_nonoverlapping( + src.add(uv_offset + y * src_stride), + frame.data_mut(1).as_mut_ptr().add(y * frame.stride(1)), + row_bytes, + ); + } + } + } _ => {} } } unsafe impl Send for D3D11Converter {} unsafe impl Sync for D3D11Converter {} + +impl Drop for D3D11Resources { + fn drop(&mut self) { + unsafe { + if let Some(handle) = self.input_shared_handle.take() + && !handle.is_invalid() + && let Err(e) = CloseHandle(handle) + { + tracing::error!("Failed to close input shared handle: {:?}", e); + } + if let Some(handle) = self.output_shared_handle.take() + && !handle.is_invalid() + && let Err(e) = CloseHandle(handle) + { + tracing::error!("Failed to close output shared handle: {:?}", e); + } + } + } +} diff --git a/crates/recording/examples/camera-benchmark.rs b/crates/recording/examples/camera-benchmark.rs index 05f00ff419..8d99d5a4e9 100644 --- a/crates/recording/examples/camera-benchmark.rs +++ b/crates/recording/examples/camera-benchmark.rs @@ -127,10 +127,7 @@ async fn run_camera_encoding_benchmark( let width = first_frame.inner.width(); let height = first_frame.inner.height(); - println!( - "\nCamera frame format: {:?} {}x{}", - input_format, width, height - ); + println!("\nCamera frame format: {input_format:?} {width}x{height}"); let output_format = Pixel::NV12; let needs_conversion = input_format != output_format; @@ -301,7 +298,10 @@ async fn run_camera_encoding_benchmark( let pipeline_latency = converted.submit_time.elapsed(); metrics.record_frame_encoded(encode_duration, pipeline_latency); } - Err(_) => {} + Err(e) => { + warn!("Encode error during drain: {}", e); + metrics.record_dropped_output(); + } } } else { break; @@ -374,8 +374,8 @@ async fn main() { println!("\n=== Frame Rate Test ==="); let (frames, fps, inter_frame_times) = run_camera_frame_rate_test(&camera.0, 3).await; - println!("Frames captured: {}", frames); - println!("Average FPS: {:.1}", fps); + println!("Frames captured: {frames}"); + println!("Average FPS: {fps:.1}"); if !inter_frame_times.is_empty() { let avg_interval: Duration = @@ -384,9 +384,9 @@ async fn main() { let min_interval = inter_frame_times.iter().min().unwrap(); println!("Inter-frame timing:"); - println!(" Average: {:?}", avg_interval); - println!(" Min: {:?}", min_interval); - println!(" Max: {:?}", max_interval); + println!(" Average: {avg_interval:?}"); + println!(" Min: {min_interval:?}"); + println!(" Max: {max_interval:?}"); let mut sorted = inter_frame_times.clone(); sorted.sort(); diff --git a/crates/recording/examples/encoding-benchmark.rs b/crates/recording/examples/encoding-benchmark.rs index 8169e30ff9..d9e62ad59a 100644 --- a/crates/recording/examples/encoding-benchmark.rs +++ b/crates/recording/examples/encoding-benchmark.rs @@ -192,7 +192,7 @@ fn benchmark_conversion_formats(config: &BenchmarkConfig) { println!("\n=== Format Conversion Benchmarks ===\n"); for (input, output, name) in formats { - println!("Testing: {}", name); + println!("Testing: {name}"); let mut cfg = config.clone(); cfg.duration_secs = 5; @@ -406,7 +406,7 @@ fn main() { "encode" => benchmark_encode_times(&config), "workers" => benchmark_worker_counts(&config), "resolutions" => benchmark_resolutions(&config), - "full" | _ => { + _ => { benchmark_conversion_formats(&config); benchmark_encode_times(&config); benchmark_worker_counts(&config); diff --git a/crates/recording/examples/recording-benchmark.rs b/crates/recording/examples/recording-benchmark.rs index 87adbe7da9..13eab946f4 100644 --- a/crates/recording/examples/recording-benchmark.rs +++ b/crates/recording/examples/recording-benchmark.rs @@ -265,7 +265,7 @@ async fn main() -> Result<(), Box> { .unwrap_or(5); stress_test_recording(cycles, config.duration_secs).await?; } - "full" | _ => { + _ => { println!("Mode: Full benchmark suite\n"); println!("--- Screen Recording ---"); diff --git a/crates/recording/src/diagnostics.rs b/crates/recording/src/diagnostics.rs new file mode 100644 index 0000000000..ceee01dd10 --- /dev/null +++ b/crates/recording/src/diagnostics.rs @@ -0,0 +1,328 @@ +#[cfg(target_os = "windows")] +mod windows_impl { + use serde::Serialize; + use specta::Type; + + #[derive(Debug, Clone, Serialize, Type)] + #[serde(rename_all = "camelCase")] + pub struct WindowsVersionInfo { + pub major: u32, + pub minor: u32, + pub build: u32, + pub display_name: String, + pub meets_requirements: bool, + pub is_windows_11: bool, + } + + #[derive(Debug, Clone, Serialize, Type)] + #[serde(rename_all = "camelCase")] + pub struct GpuInfoDiag { + pub vendor: String, + pub description: String, + pub dedicated_video_memory_mb: f64, + pub adapter_index: u32, + pub is_software_adapter: bool, + pub is_basic_render_driver: bool, + pub supports_hardware_encoding: bool, + } + + #[derive(Debug, Clone, Serialize, Type)] + #[serde(rename_all = "camelCase")] + pub struct AllGpusInfo { + pub gpus: Vec, + pub primary_gpu_index: Option, + pub is_multi_gpu_system: bool, + pub has_discrete_gpu: bool, + } + + #[derive(Debug, Clone, Serialize, Type)] + #[serde(rename_all = "camelCase")] + pub struct RenderingStatus { + pub is_using_software_rendering: bool, + pub is_using_basic_render_driver: bool, + pub hardware_encoding_available: bool, + pub warning_message: Option, + } + + #[derive(Debug, Clone, Serialize, Type)] + #[serde(rename_all = "camelCase")] + pub struct SystemDiagnostics { + pub windows_version: Option, + pub gpu_info: Option, + pub all_gpus: Option, + pub rendering_status: RenderingStatus, + pub available_encoders: Vec, + pub graphics_capture_supported: bool, + #[serde(rename = "d3D11VideoProcessorAvailable")] + pub d3d11_video_processor_available: bool, + } + + pub fn collect_diagnostics() -> SystemDiagnostics { + let windows_version = get_windows_version_info(); + let gpu_info = get_gpu_info(); + let all_gpus = get_all_gpus_info(); + let rendering_status = get_rendering_status(&gpu_info); + let available_encoders = get_available_encoders(); + let graphics_capture_supported = check_graphics_capture_support(); + let d3d11_video_processor_available = check_d3d11_video_processor(); + + tracing::info!("System Diagnostics:"); + if let Some(ref ver) = windows_version { + tracing::info!(" Windows: {}", ver.display_name); + } + if let Some(ref gpu) = gpu_info { + tracing::info!( + " Primary GPU: {} ({}) - Software: {}, BasicRender: {}", + gpu.description, + gpu.vendor, + gpu.is_software_adapter, + gpu.is_basic_render_driver + ); + } + if let Some(ref all) = all_gpus { + tracing::info!( + " GPU Count: {}, Multi-GPU: {}, Has Discrete: {}", + all.gpus.len(), + all.is_multi_gpu_system, + all.has_discrete_gpu + ); + } + tracing::info!( + " Rendering: SoftwareRendering={}, HardwareEncoding={}", + rendering_status.is_using_software_rendering, + rendering_status.hardware_encoding_available + ); + if let Some(ref warning) = rendering_status.warning_message { + tracing::warn!(" Warning: {}", warning); + } + tracing::info!(" Encoders: {:?}", available_encoders); + tracing::info!(" Graphics Capture: {}", graphics_capture_supported); + tracing::info!( + " D3D11 Video Processor: {}", + d3d11_video_processor_available + ); + + SystemDiagnostics { + windows_version, + gpu_info, + all_gpus, + rendering_status, + available_encoders, + graphics_capture_supported, + d3d11_video_processor_available, + } + } + + fn get_windows_version_info() -> Option { + scap_direct3d::WindowsVersion::detect().map(|v| WindowsVersionInfo { + major: v.major, + minor: v.minor, + build: v.build, + display_name: v.display_name(), + meets_requirements: v.meets_minimum_requirements(), + is_windows_11: v.is_windows_11(), + }) + } + + fn gpu_info_to_diag(info: &cap_frame_converter::GpuInfo) -> GpuInfoDiag { + GpuInfoDiag { + vendor: info.vendor_name().to_string(), + description: info.description.clone(), + dedicated_video_memory_mb: (info.dedicated_video_memory / (1024 * 1024)) as f64, + adapter_index: info.adapter_index, + is_software_adapter: info.is_software_adapter, + is_basic_render_driver: info.is_basic_render_driver(), + supports_hardware_encoding: info.supports_hardware_encoding(), + } + } + + fn get_gpu_info() -> Option { + cap_frame_converter::detect_primary_gpu().map(gpu_info_to_diag) + } + + fn get_all_gpus_info() -> Option { + let all_gpus = cap_frame_converter::get_all_gpus(); + + if all_gpus.is_empty() { + return None; + } + + let gpus: Vec = all_gpus.iter().map(gpu_info_to_diag).collect(); + + let primary_gpu = cap_frame_converter::detect_primary_gpu(); + let primary_gpu_index = primary_gpu.and_then(|primary| { + all_gpus + .iter() + .position(|g| g.adapter_index == primary.adapter_index) + .map(|idx| idx as u32) + }); + + let has_discrete = all_gpus.iter().any(|g| { + matches!( + g.vendor, + cap_frame_converter::GpuVendor::Nvidia + | cap_frame_converter::GpuVendor::Amd + | cap_frame_converter::GpuVendor::Qualcomm + | cap_frame_converter::GpuVendor::Arm + ) && !g.is_software_adapter + }); + + Some(AllGpusInfo { + is_multi_gpu_system: gpus.len() > 1, + has_discrete_gpu: has_discrete, + primary_gpu_index, + gpus, + }) + } + + fn get_rendering_status(gpu_info: &Option) -> RenderingStatus { + let (is_software, is_basic_render, hw_encoding, warning) = match gpu_info { + Some(gpu) => { + let is_basic = gpu.is_basic_render_driver; + let is_software = gpu.is_software_adapter; + let hw_available = gpu.supports_hardware_encoding; + + let warning = if is_basic { + Some( + "Microsoft Basic Render Driver detected. This may indicate missing GPU drivers or a remote desktop session. Recording will use software encoding which may impact performance." + .to_string(), + ) + } else if is_software { + Some( + "Software rendering is active. Hardware GPU acceleration is not available. Update your graphics drivers for better performance." + .to_string(), + ) + } else if !hw_available { + Some( + "Hardware encoding may not be available on this GPU. Software encoding will be used as a fallback." + .to_string(), + ) + } else { + None + }; + + (is_software, is_basic, hw_available, warning) + } + None => ( + true, + false, + false, + Some("No GPU detected. Recording will use software encoding.".to_string()), + ), + }; + + RenderingStatus { + is_using_software_rendering: is_software, + is_using_basic_render_driver: is_basic_render, + hardware_encoding_available: hw_encoding, + warning_message: warning, + } + } + + fn get_available_encoders() -> Vec { + let candidates = [ + "h264_nvenc", + "h264_qsv", + "h264_amf", + "h264_mf", + "libx264", + "hevc_nvenc", + "hevc_qsv", + "hevc_amf", + "hevc_mf", + "libx265", + ]; + + candidates + .iter() + .filter(|name| ffmpeg::encoder::find_by_name(name).is_some()) + .map(|s| s.to_string()) + .collect() + } + + fn check_graphics_capture_support() -> bool { + scap_direct3d::is_supported().unwrap_or(false) + } + + fn check_d3d11_video_processor() -> bool { + use cap_frame_converter::ConversionConfig; + + let test_config = ConversionConfig::new( + ffmpeg::format::Pixel::BGRA, + 1920, + 1080, + ffmpeg::format::Pixel::NV12, + 1920, + 1080, + ); + + match cap_frame_converter::D3D11Converter::new(test_config) { + Ok(converter) => { + tracing::debug!( + "D3D11 video processor check passed: {} ({})", + converter.gpu_info().description, + converter.gpu_info().vendor_name() + ); + true + } + Err(e) => { + tracing::warn!("D3D11 video processor check failed: {e:?}"); + false + } + } + } +} + +#[cfg(target_os = "macos")] +mod macos_impl { + use serde::Serialize; + use specta::Type; + + #[derive(Debug, Clone, Serialize, Type)] + #[serde(rename_all = "camelCase")] + pub struct MacOSVersionInfo { + pub display_name: String, + } + + #[derive(Debug, Clone, Serialize, Type)] + #[serde(rename_all = "camelCase")] + pub struct SystemDiagnostics { + pub macos_version: Option, + pub available_encoders: Vec, + pub screen_capture_supported: bool, + } + + pub fn collect_diagnostics() -> SystemDiagnostics { + let available_encoders = get_available_encoders(); + + tracing::info!("System Diagnostics:"); + tracing::info!(" Encoders: {:?}", available_encoders); + + SystemDiagnostics { + macos_version: None, + available_encoders, + screen_capture_supported: true, + } + } + + fn get_available_encoders() -> Vec { + let candidates = [ + "h264_videotoolbox", + "libx264", + "hevc_videotoolbox", + "libx265", + ]; + + candidates + .iter() + .filter(|name| ffmpeg::encoder::find_by_name(name).is_some()) + .map(|s| s.to_string()) + .collect() + } +} + +#[cfg(target_os = "windows")] +pub use windows_impl::*; + +#[cfg(target_os = "macos")] +pub use macos_impl::*; diff --git a/crates/recording/src/lib.rs b/crates/recording/src/lib.rs index e68b92d15e..dc1670b27d 100644 --- a/crates/recording/src/lib.rs +++ b/crates/recording/src/lib.rs @@ -1,6 +1,7 @@ pub mod benchmark; mod capture_pipeline; pub mod cursor; +pub mod diagnostics; pub mod feeds; pub mod fragmentation; pub mod instant_recording; diff --git a/crates/recording/src/output_pipeline/win.rs b/crates/recording/src/output_pipeline/win.rs index a9c5a756ec..18d7107906 100644 --- a/crates/recording/src/output_pipeline/win.rs +++ b/crates/recording/src/output_pipeline/win.rs @@ -113,7 +113,9 @@ impl Muxer for WindowsMuxer { let output_size = config.output_size.unwrap_or(input_size); let fragmented = config.fragmented; let frag_duration_us = config.frag_duration_us; - let (video_tx, video_rx) = sync_channel::>(8); + let queue_depth = ((config.frame_rate as f32 / 30.0 * 5.0).ceil() as usize).clamp(3, 12); + let (video_tx, video_rx) = + sync_channel::>(queue_depth); let mut output = ffmpeg::format::output(&output_path)?; @@ -187,6 +189,12 @@ impl Muxer for WindowsMuxer { config.bitrate_multiplier, ) { Ok(encoder) => { + if let Err(e) = encoder.validate() { + return fallback(Some(format!( + "Hardware encoder validation failed: {e}" + ))); + } + let width = match u32::try_from(output_size.Width) { Ok(width) if width > 0 => width, _ => { @@ -258,36 +266,57 @@ impl Muxer for WindowsMuxer { either::Left((mut encoder, mut muxer)) => { trace!("Running native encoder"); let mut first_timestamp: Option = None; - encoder - .run( - Arc::new(AtomicBool::default()), - || { - let Ok(Some((frame, timestamp))) = video_rx.recv() else { - trace!("No more frames available"); - return Ok(None); - }; - - let relative = if let Some(first) = first_timestamp { - timestamp.checked_sub(first).unwrap_or(Duration::ZERO) - } else { - first_timestamp = Some(timestamp); - Duration::ZERO - }; - let frame_time = duration_to_timespan(relative); - - Ok(Some((frame.texture().clone(), frame_time))) - }, - |output_sample| { - let mut output = output.lock().unwrap(); + let result = encoder.run( + Arc::new(AtomicBool::default()), + || { + let Ok(Some((frame, timestamp))) = video_rx.recv() else { + trace!("No more frames available"); + return Ok(None); + }; + + let relative = if let Some(first) = first_timestamp { + timestamp.saturating_sub(first) + } else { + first_timestamp = Some(timestamp); + Duration::ZERO + }; + let frame_time = duration_to_timespan(relative); - let _ = muxer - .write_sample(&output_sample, &mut output) - .map_err(|e| format!("WriteSample: {e}")); + Ok(Some((frame.texture().clone(), frame_time))) + }, + |output_sample| { + let Ok(mut output) = output.lock() else { + tracing::error!("Failed to lock output mutex - poisoned"); + return Ok(()); + }; - Ok(()) - }, - ) - .context("run native encoder") + if let Err(e) = muxer.write_sample(&output_sample, &mut output) { + tracing::error!("WriteSample failed: {e}"); + } + + Ok(()) + }, + ); + + match result { + Ok(health_status) => { + debug!( + "Hardware encoder completed: {} frames encoded", + health_status.total_frames_encoded + ); + Ok(()) + } + Err(e) => { + if e.should_fallback() { + error!( + "Hardware encoder failed with recoverable error, marking for software fallback: {}", + e + ); + encoder_preferences.force_software_only(); + } + Err(anyhow!("Hardware encoder error: {}", e)) + } + } } either::Right(mut encoder) => { while let Ok(Some((frame, time))) = video_rx.recv() { @@ -433,6 +462,7 @@ pub struct WindowsCameraMuxerConfig { pub output_height: Option, pub fragmented: bool, pub frag_duration_us: i64, + pub encoder_preferences: crate::capture_pipeline::EncoderPreferences, } impl Default for WindowsCameraMuxerConfig { @@ -441,6 +471,7 @@ impl Default for WindowsCameraMuxerConfig { output_height: None, fragmented: false, frag_duration_us: 2_000_000, + encoder_preferences: crate::capture_pipeline::EncoderPreferences::new(), } } } @@ -499,6 +530,7 @@ impl Muxer for WindowsCameraMuxer { { let output = output.clone(); + let encoder_preferences = config.encoder_preferences; tasks.spawn_thread("windows-camera-encoder", move || { cap_mediafoundation_utils::thread_init(); @@ -525,132 +557,249 @@ impl Muxer for WindowsCameraMuxer { let input_format = first_frame.0.dxgi_format(); - let encoder_result = cap_enc_mediafoundation::H264Encoder::new_with_scaled_output( - &d3d_device, - input_format, - input_size, - output_size, - frame_rate, - bitrate_multiplier, - ); - - let (mut encoder, mut muxer) = match encoder_result { - Ok(encoder) => { - let muxer = { - let mut output_guard = match output.lock() { - Ok(guard) => guard, - Err(poisoned) => { - let msg = format!("Failed to lock output mutex: {poisoned}"); - let _ = ready_tx.send(Err(anyhow!("{}", msg))); - return Err(anyhow!("{}", msg)); - } - }; + let encoder = (|| { + let fallback = |reason: Option| { + encoder_preferences.force_software_only(); + if let Some(reason) = reason.as_ref() { + error!( + "Falling back to software H264 encoder for camera: {reason}" + ); + } else { + info!("Using software H264 encoder for camera"); + } - cap_mediafoundation_ffmpeg::H264StreamMuxer::new( - &mut output_guard, - cap_mediafoundation_ffmpeg::MuxerConfig { - width: output_width, - height: output_height, - fps: frame_rate, - bitrate: encoder.bitrate(), - fragmented, - frag_duration_us, - }, - ) + let mut output_guard = match output.lock() { + Ok(guard) => guard, + Err(poisoned) => { + return Err(anyhow!( + "CameraSoftwareEncoder: failed to lock output mutex: {}", + poisoned + )); + } }; - match muxer { - Ok(muxer) => (encoder, muxer), - Err(err) => { - let msg = format!("Failed to create muxer: {err}"); - let _ = ready_tx.send(Err(anyhow!("{}", msg))); - return Err(anyhow!("{}", msg)); + cap_enc_ffmpeg::h264::H264Encoder::builder(video_config) + .with_output_size(output_width, output_height) + .and_then(|builder| builder.build(&mut output_guard)) + .map(either::Right) + .map_err(|e| anyhow!("CameraSoftwareEncoder/{e}")) + }; + + if encoder_preferences.should_force_software() { + return fallback(None); + } + + match cap_enc_mediafoundation::H264Encoder::new_with_scaled_output( + &d3d_device, + input_format, + input_size, + output_size, + frame_rate, + bitrate_multiplier, + ) { + Ok(encoder) => { + if let Err(e) = encoder.validate() { + return fallback(Some(format!( + "Camera hardware encoder validation failed: {e}" + ))); + } + + let muxer = { + let mut output_guard = match output.lock() { + Ok(guard) => guard, + Err(poisoned) => { + return fallback(Some(format!( + "Failed to lock output mutex: {poisoned}" + ))); + } + }; + + cap_mediafoundation_ffmpeg::H264StreamMuxer::new( + &mut output_guard, + cap_mediafoundation_ffmpeg::MuxerConfig { + width: output_width, + height: output_height, + fps: frame_rate, + bitrate: encoder.bitrate(), + fragmented, + frag_duration_us, + }, + ) + }; + + match muxer { + Ok(muxer) => Ok(either::Left((encoder, muxer))), + Err(err) => fallback(Some(err.to_string())), } } + Err(err) => fallback(Some(err.to_string())), } - Err(err) => { - let msg = format!("Failed to create H264 encoder: {err}"); - let _ = ready_tx.send(Err(anyhow!("{}", msg))); - return Err(anyhow!("{}", msg)); + })(); + + let encoder = match encoder { + Ok(encoder) => { + if ready_tx.send(Ok(())).is_err() { + error!("Failed to send ready signal - receiver dropped"); + return Ok(()); + } + encoder + } + Err(e) => { + error!("Camera encoder setup failed: {:#}", e); + let _ = ready_tx.send(Err(anyhow!("{e}"))); + return Err(anyhow!("{e}")); } }; - if ready_tx.send(Ok(())).is_err() { - error!("Failed to send ready signal - receiver dropped"); - return Ok(()); - } + match encoder { + either::Left((mut encoder, mut muxer)) => { + info!( + "Windows camera encoder started (hardware): {:?} {}x{} -> NV12 {}x{} @ {}fps", + input_format, + input_size.Width, + input_size.Height, + output_size.Width, + output_size.Height, + frame_rate + ); - info!( - "Windows camera encoder started: {:?} {}x{} -> NV12 {}x{} @ {}fps", - input_format, - input_size.Width, - input_size.Height, - output_size.Width, - output_size.Height, - frame_rate - ); - - let mut first_timestamp: Option = None; - let mut frame_count = 0u64; - - let mut process_frame = |frame: NativeCameraFrame, - timestamp: Duration| - -> windows::core::Result< - Option<( - windows::Win32::Graphics::Direct3D11::ID3D11Texture2D, - TimeSpan, - )>, - > { - let relative = if let Some(first) = first_timestamp { - timestamp.checked_sub(first).unwrap_or(Duration::ZERO) - } else { - first_timestamp = Some(timestamp); - Duration::ZERO - }; + let mut first_timestamp: Option = None; + let mut frame_count = 0u64; + + let mut process_frame = |frame: NativeCameraFrame, + timestamp: Duration| + -> windows::core::Result< + Option<( + windows::Win32::Graphics::Direct3D11::ID3D11Texture2D, + TimeSpan, + )>, + > { + let relative = if let Some(first) = first_timestamp { + timestamp.saturating_sub(first) + } else { + first_timestamp = Some(timestamp); + Duration::ZERO + }; - let texture = upload_mf_buffer_to_texture(&d3d_device, &frame)?; - Ok(Some((texture, duration_to_timespan(relative)))) - }; + let texture = upload_mf_buffer_to_texture(&d3d_device, &frame)?; + Ok(Some((texture, duration_to_timespan(relative)))) + }; - if let Ok(Some((texture, frame_time))) = process_frame(first_frame.0, first_frame.1) - { - encoder - .run( - Arc::new(AtomicBool::default()), - || { - if frame_count > 0 { - let Ok(Some((frame, timestamp))) = video_rx.recv() else { - trace!("No more camera frames available"); - return Ok(None); - }; + if let Ok(Some((texture, frame_time))) = + process_frame(first_frame.0, first_frame.1) + { + let result = encoder.run( + Arc::new(AtomicBool::default()), + || { + if frame_count > 0 { + let Ok(Some((frame, timestamp))) = video_rx.recv() else { + trace!("No more camera frames available"); + return Ok(None); + }; + frame_count += 1; + if frame_count.is_multiple_of(30) { + debug!( + "Windows camera encoder: processed {} frames", + frame_count + ); + } + return process_frame(frame, timestamp); + } frame_count += 1; - if frame_count.is_multiple_of(30) { - debug!( - "Windows camera encoder: processed {} frames", - frame_count + Ok(Some((texture.clone(), frame_time))) + }, + |output_sample| { + let mut output = output.lock().unwrap(); + if let Err(e) = muxer.write_sample(&output_sample, &mut output) + { + tracing::error!("Camera WriteSample failed: {e}"); + } + Ok(()) + }, + ); + + match result { + Ok(health_status) => { + info!( + "Windows camera encoder finished (hardware): {} frames encoded", + health_status.total_frames_encoded + ); + } + Err(e) => { + if e.should_fallback() { + error!( + "Camera hardware encoder failed with recoverable error, marking for software fallback: {}", + e ); + encoder_preferences.force_software_only(); } - return process_frame(frame, timestamp); + return Err(anyhow!("Camera hardware encoder error: {}", e)); } + } + } + + Ok(()) + } + either::Right(mut encoder) => { + info!( + "Windows camera encoder started (software): {}x{} -> {}x{} @ {}fps", + video_config.width, + video_config.height, + output_width, + output_height, + frame_rate + ); + + let mut first_timestamp: Option = None; + let mut frame_count = 0u64; + + let mut process_frame = + |frame: NativeCameraFrame, + timestamp: Duration| + -> anyhow::Result> { + let relative = if let Some(first) = first_timestamp { + timestamp.saturating_sub(first) + } else { + first_timestamp = Some(timestamp); + Duration::ZERO + }; + + let ffmpeg_frame = camera_frame_to_ffmpeg(&frame)?; + + let Ok(mut output_guard) = output.lock() else { + return Ok(None); + }; + + encoder + .queue_frame(ffmpeg_frame, relative, &mut output_guard) + .context("queue camera frame")?; + + Ok(Some(relative)) + }; + + if process_frame(first_frame.0, first_frame.1)?.is_some() { + frame_count += 1; + } + + while let Ok(Some((frame, timestamp))) = video_rx.recv() { + if process_frame(frame, timestamp)?.is_some() { frame_count += 1; - Ok(Some((texture.clone(), frame_time))) - }, - |output_sample| { - let mut output = output.lock().unwrap(); - let _ = muxer - .write_sample(&output_sample, &mut output) - .map_err(|e| format!("WriteSample: {e}")); - Ok(()) - }, - ) - .context("run camera encoder")?; - } + if frame_count.is_multiple_of(30) { + debug!( + "Windows camera encoder (software): processed {} frames", + frame_count + ); + } + } + } - info!( - "Windows camera encoder finished: {} frames encoded", - frame_count - ); - Ok(()) + info!( + "Windows camera encoder finished (software): {} frames encoded", + frame_count + ); + Ok(()) + } + } }); } @@ -722,18 +871,108 @@ impl AudioMuxer for WindowsCameraMuxer { fn convert_uyvy_to_yuyv(src: &[u8], width: u32, height: u32) -> Vec { let total_bytes = (width * height * 2) as usize; + let src_len = src.len().min(total_bytes); let mut dst = vec![0u8; total_bytes]; - for i in (0..src.len().min(total_bytes)).step_by(4) { - if i + 3 < src.len() && i + 3 < total_bytes { + #[cfg(target_arch = "x86_64")] + { + if is_x86_feature_detected!("ssse3") { + unsafe { + convert_uyvy_to_yuyv_ssse3(src, &mut dst, src_len); + } + return dst; + } + } + + convert_uyvy_to_yuyv_scalar(src, &mut dst, src_len); + dst +} + +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "ssse3")] +unsafe fn convert_uyvy_to_yuyv_ssse3(src: &[u8], dst: &mut [u8], len: usize) { + use std::arch::x86_64::*; + + unsafe { + let shuffle_mask = _mm_setr_epi8(1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); + + let mut i = 0; + let simd_end = len & !15; + + while i < simd_end { + let chunk = _mm_loadu_si128(src.as_ptr().add(i) as *const __m128i); + let shuffled = _mm_shuffle_epi8(chunk, shuffle_mask); + _mm_storeu_si128(dst.as_mut_ptr().add(i) as *mut __m128i, shuffled); + i += 16; + } + + convert_uyvy_to_yuyv_scalar(&src[i..], &mut dst[i..], len - i); + } +} + +fn convert_uyvy_to_yuyv_scalar(src: &[u8], dst: &mut [u8], len: usize) { + for i in (0..len).step_by(4) { + if i + 3 < src.len() && i + 3 < dst.len() { dst[i] = src[i + 1]; dst[i + 1] = src[i]; dst[i + 2] = src[i + 3]; dst[i + 3] = src[i + 2]; } } +} - dst +pub fn camera_frame_to_ffmpeg(frame: &NativeCameraFrame) -> anyhow::Result { + use cap_mediafoundation_utils::IMFMediaBufferExt; + + let ffmpeg_format = match frame.pixel_format { + cap_camera_windows::PixelFormat::NV12 => ffmpeg::format::Pixel::NV12, + cap_camera_windows::PixelFormat::YUYV422 => ffmpeg::format::Pixel::YUYV422, + cap_camera_windows::PixelFormat::UYVY422 => ffmpeg::format::Pixel::UYVY422, + other => anyhow::bail!("Unsupported camera pixel format: {:?}", other), + }; + + let buffer_guard = frame + .buffer + .lock() + .map_err(|_| anyhow!("Failed to lock camera buffer"))?; + let lock = buffer_guard + .lock() + .map_err(|e| anyhow!("Failed to lock MF buffer: {:?}", e))?; + let data = &*lock; + + let converted_data_storage; + let (final_data, final_format): (&[u8], ffmpeg::format::Pixel) = + if frame.pixel_format == cap_camera_windows::PixelFormat::UYVY422 { + converted_data_storage = convert_uyvy_to_yuyv(data, frame.width, frame.height); + ( + converted_data_storage.as_slice(), + ffmpeg::format::Pixel::YUYV422, + ) + } else { + (data, ffmpeg_format) + }; + + let mut ffmpeg_frame = ffmpeg::frame::Video::new(final_format, frame.width, frame.height); + + match final_format { + ffmpeg::format::Pixel::NV12 => { + let y_size = (frame.width * frame.height) as usize; + let uv_size = y_size / 2; + if final_data.len() >= y_size + uv_size { + ffmpeg_frame.data_mut(0)[..y_size].copy_from_slice(&final_data[..y_size]); + ffmpeg_frame.data_mut(1)[..uv_size].copy_from_slice(&final_data[y_size..]); + } + } + ffmpeg::format::Pixel::YUYV422 => { + let size = (frame.width * frame.height * 2) as usize; + if final_data.len() >= size { + ffmpeg_frame.data_mut(0)[..size].copy_from_slice(&final_data[..size]); + } + } + _ => {} + } + + Ok(ffmpeg_frame) } pub fn upload_mf_buffer_to_texture( @@ -761,14 +1000,10 @@ pub fn upload_mf_buffer_to_texture( let lock = buffer_guard.lock()?; let original_data = &*lock; - let converted_buffer: Option>; + let converted_buffer_storage; let data: &[u8] = if frame.pixel_format == cap_camera_windows::PixelFormat::UYVY422 { - converted_buffer = Some(convert_uyvy_to_yuyv( - original_data, - frame.width, - frame.height, - )); - converted_buffer.as_ref().unwrap() + converted_buffer_storage = convert_uyvy_to_yuyv(original_data, frame.width, frame.height); + converted_buffer_storage.as_slice() } else { original_data }; diff --git a/crates/recording/src/output_pipeline/win_segmented.rs b/crates/recording/src/output_pipeline/win_segmented.rs index bdb8393041..5c98f790a4 100644 --- a/crates/recording/src/output_pipeline/win_segmented.rs +++ b/crates/recording/src/output_pipeline/win_segmented.rs @@ -354,7 +354,9 @@ impl WindowsSegmentedMuxer { }; let output_size = self.output_size.unwrap_or(input_size); - let (video_tx, video_rx) = sync_channel::>(8); + let queue_depth = ((self.frame_rate as f32 / 30.0 * 5.0).ceil() as usize).clamp(3, 12); + let (video_tx, video_rx) = + sync_channel::>(queue_depth); let (ready_tx, ready_rx) = sync_channel::>(1); let output = ffmpeg::format::output(&segment_path)?; let output = Arc::new(Mutex::new(output)); @@ -422,6 +424,12 @@ impl WindowsSegmentedMuxer { bitrate_multiplier, ) { Ok(encoder) => { + if let Err(e) = encoder.validate() { + return fallback(Some(format!( + "Hardware encoder validation failed: {e}" + ))); + } + let width = match u32::try_from(output_size.Width) { Ok(width) if width > 0 => width, _ => { @@ -493,36 +501,63 @@ impl WindowsSegmentedMuxer { either::Left((mut encoder, mut muxer)) => { trace!("Running native encoder for segment"); let mut first_timestamp: Option = None; - encoder - .run( - Arc::new(AtomicBool::default()), - || { - let Ok(Some((frame, timestamp))) = video_rx.recv() else { - trace!("No more frames available for segment"); - return Ok(None); - }; - - let relative = if let Some(first) = first_timestamp { - timestamp.checked_sub(first).unwrap_or(Duration::ZERO) - } else { - first_timestamp = Some(timestamp); - Duration::ZERO - }; - let frame_time = duration_to_timespan(relative); - - Ok(Some((frame.texture().clone(), frame_time))) - }, - |output_sample| { - let mut output = output_clone.lock().unwrap(); - - let _ = muxer - .write_sample(&output_sample, &mut output) - .map_err(|e| format!("WriteSample: {e}")); - - Ok(()) - }, - ) - .context("run native encoder for segment") + let result = encoder.run( + Arc::new(AtomicBool::default()), + || { + let Ok(Some((frame, timestamp))) = video_rx.recv() else { + trace!("No more frames available for segment"); + return Ok(None); + }; + + let relative = if let Some(first) = first_timestamp { + timestamp.checked_sub(first).unwrap_or(Duration::ZERO) + } else { + first_timestamp = Some(timestamp); + Duration::ZERO + }; + let frame_time = duration_to_timespan(relative); + + Ok(Some((frame.texture().clone(), frame_time))) + }, + |output_sample| { + let mut output = match output_clone.lock() { + Ok(guard) => guard, + Err(e) => { + error!("Failed to lock output mutex: {e}"); + return Err(windows::core::Error::new( + windows::core::HRESULT(0x80004005u32 as i32), + format!("Mutex poisoned: {e}"), + )); + } + }; + + if let Err(e) = muxer.write_sample(&output_sample, &mut output) { + warn!("WriteSample failed: {e}"); + } + + Ok(()) + }, + ); + + match result { + Ok(health_status) => { + debug!( + "Hardware encoder completed for segment: {} frames encoded", + health_status.total_frames_encoded + ); + Ok(()) + } + Err(e) => { + if e.should_fallback() { + error!( + "Hardware encoder failed with recoverable error, marking for software fallback: {}", + e + ); + encoder_preferences.force_software_only(); + } + Err(anyhow!("Hardware encoder error: {}", e)) + } + } } either::Right(mut encoder) => { while let Ok(Some((frame, time))) = video_rx.recv() { diff --git a/crates/recording/src/output_pipeline/win_segmented_camera.rs b/crates/recording/src/output_pipeline/win_segmented_camera.rs index 67878cf3a2..71fbf6fcd6 100644 --- a/crates/recording/src/output_pipeline/win_segmented_camera.rs +++ b/crates/recording/src/output_pipeline/win_segmented_camera.rs @@ -147,6 +147,7 @@ pub struct WindowsSegmentedCameraMuxer { video_config: VideoInfo, output_height: Option, + encoder_preferences: crate::capture_pipeline::EncoderPreferences, pause: PauseTracker, frame_drops: FrameDropTracker, @@ -155,6 +156,7 @@ pub struct WindowsSegmentedCameraMuxer { pub struct WindowsSegmentedCameraMuxerConfig { pub output_height: Option, pub segment_duration: Duration, + pub encoder_preferences: crate::capture_pipeline::EncoderPreferences, } impl Default for WindowsSegmentedCameraMuxerConfig { @@ -162,6 +164,7 @@ impl Default for WindowsSegmentedCameraMuxerConfig { Self { output_height: None, segment_duration: Duration::from_secs(3), + encoder_preferences: crate::capture_pipeline::EncoderPreferences::new(), } } } @@ -195,6 +198,7 @@ impl Muxer for WindowsSegmentedCameraMuxer { current_state: None, video_config, output_height: config.output_height, + encoder_preferences: config.encoder_preferences, pause: PauseTracker::new(pause_flag), frame_drops: FrameDropTracker::new(), }) @@ -341,6 +345,8 @@ impl WindowsSegmentedCameraMuxer { } fn create_segment(&mut self, first_frame: &NativeCameraFrame) -> anyhow::Result<()> { + use crate::output_pipeline::win::camera_frame_to_ffmpeg; + let segment_path = self.current_segment_path(); let input_size = SizeInt32 { @@ -361,6 +367,8 @@ impl WindowsSegmentedCameraMuxer { let frame_rate = self.video_config.fps(); let bitrate_multiplier = 0.2f32; let input_format = first_frame.dxgi_format(); + let video_config = self.video_config; + let encoder_preferences = self.encoder_preferences.clone(); let (video_tx, video_rx) = sync_channel::>(30); let (ready_tx, ready_rx) = sync_channel::>(1); @@ -381,85 +389,183 @@ impl WindowsSegmentedCameraMuxer { } }; - let encoder_result = cap_enc_mediafoundation::H264Encoder::new_with_scaled_output( - &d3d_device, - input_format, - input_size, - output_size, - frame_rate, - bitrate_multiplier, - ); - - let (mut encoder, mut muxer) = match encoder_result { - Ok(encoder) => { - let muxer = { - let mut output_guard = match output_clone.lock() { - Ok(guard) => guard, - Err(poisoned) => { - let _ = ready_tx.send(Err(anyhow!( - "Failed to lock output mutex: {poisoned}" - ))); - return Err(anyhow!( - "Failed to lock output mutex: {}", - poisoned - )); - } - }; + let encoder = (|| { + let fallback = |reason: Option| { + encoder_preferences.force_software_only(); + if let Some(reason) = reason.as_ref() { + error!( + "Falling back to software H264 encoder for camera segment: {reason}" + ); + } else { + info!("Using software H264 encoder for camera segment"); + } - cap_mediafoundation_ffmpeg::H264StreamMuxer::new( - &mut output_guard, - cap_mediafoundation_ffmpeg::MuxerConfig { - width: output_width, - height: output_height, - fps: frame_rate, - bitrate: encoder.bitrate(), - fragmented: false, - frag_duration_us: 0, - }, - ) + let mut output_guard = match output_clone.lock() { + Ok(guard) => guard, + Err(poisoned) => { + return Err(anyhow!( + "CameraSegmentSoftwareEncoder: failed to lock output mutex: {}", + poisoned + )); + } }; - match muxer { - Ok(muxer) => (encoder, muxer), - Err(err) => { - let _ = - ready_tx.send(Err(anyhow!("Failed to create muxer: {err}"))); - return Err(anyhow!("Failed to create muxer: {err}")); + cap_enc_ffmpeg::h264::H264Encoder::builder(video_config) + .with_output_size(output_width, output_height) + .and_then(|builder| builder.build(&mut output_guard)) + .map(either::Right) + .map_err(|e| anyhow!("CameraSegmentSoftwareEncoder/{e}")) + }; + + if encoder_preferences.should_force_software() { + return fallback(None); + } + + match cap_enc_mediafoundation::H264Encoder::new_with_scaled_output( + &d3d_device, + input_format, + input_size, + output_size, + frame_rate, + bitrate_multiplier, + ) { + Ok(encoder) => { + if let Err(e) = encoder.validate() { + return fallback(Some(format!( + "Camera hardware encoder validation failed: {e}" + ))); } + + let muxer = { + let mut output_guard = match output_clone.lock() { + Ok(guard) => guard, + Err(poisoned) => { + return fallback(Some(format!( + "Failed to lock output mutex: {poisoned}" + ))); + } + }; + + cap_mediafoundation_ffmpeg::H264StreamMuxer::new( + &mut output_guard, + cap_mediafoundation_ffmpeg::MuxerConfig { + width: output_width, + height: output_height, + fps: frame_rate, + bitrate: encoder.bitrate(), + fragmented: false, + frag_duration_us: 0, + }, + ) + }; + + match muxer { + Ok(muxer) => Ok(either::Left((encoder, muxer))), + Err(err) => fallback(Some(err.to_string())), + } + } + Err(err) => fallback(Some(err.to_string())), + } + })(); + + let encoder = match encoder { + Ok(encoder) => { + if ready_tx.send(Ok(())).is_err() { + error!("Failed to send ready signal - receiver dropped"); + return Ok(()); } + encoder } - Err(err) => { - let _ = ready_tx.send(Err(anyhow!("Failed to create H264 encoder: {err}"))); - return Err(anyhow!("Failed to create H264 encoder: {err}")); + Err(e) => { + error!("Camera segment encoder setup failed: {:#}", e); + let _ = ready_tx.send(Err(anyhow!("{e}"))); + return Err(anyhow!("{e}")); } }; - if ready_tx.send(Ok(())).is_err() { - error!("Failed to send ready signal - receiver dropped"); - return Ok(()); - } + match encoder { + either::Left((mut encoder, mut muxer)) => { + info!( + "Camera segment encoder started (hardware): {:?} {}x{} -> NV12 {}x{} @ {}fps", + input_format, + input_size.Width, + input_size.Height, + output_size.Width, + output_size.Height, + frame_rate + ); - info!( - "Camera segment encoder started: {:?} {}x{} -> NV12 {}x{} @ {}fps", - input_format, - input_size.Width, - input_size.Height, - output_size.Width, - output_size.Height, - frame_rate - ); - - let mut first_timestamp: Option = None; - - encoder - .run( - Arc::new(AtomicBool::default()), - || { - let Ok(Some((frame, timestamp))) = video_rx.recv() else { - trace!("No more camera frames available for segment"); - return Ok(None); - }; + let mut first_timestamp: Option = None; + + let result = encoder.run( + Arc::new(AtomicBool::default()), + || { + let Ok(Some((frame, timestamp))) = video_rx.recv() else { + trace!("No more camera frames available for segment"); + return Ok(None); + }; + + let relative = if let Some(first) = first_timestamp { + timestamp.checked_sub(first).unwrap_or(Duration::ZERO) + } else { + first_timestamp = Some(timestamp); + Duration::ZERO + }; + + let texture = upload_mf_buffer_to_texture(&d3d_device, &frame)?; + Ok(Some((texture, duration_to_timespan(relative)))) + }, + |output_sample| { + let mut output = output_clone.lock().map_err(|e| { + windows::core::Error::new( + windows::core::HRESULT(-1), + format!("Mutex poisoned: {e}"), + ) + })?; + muxer + .write_sample(&output_sample, &mut output) + .map_err(|e| { + windows::core::Error::new( + windows::core::HRESULT(-1), + format!("WriteSample: {e}"), + ) + }) + }, + ); + match result { + Ok(health_status) => { + info!( + "Camera segment encoder finished (hardware): {} frames encoded", + health_status.total_frames_encoded + ); + Ok(()) + } + Err(e) => { + if e.should_fallback() { + error!( + "Camera hardware encoder failed with recoverable error, marking for software fallback: {}", + e + ); + encoder_preferences.force_software_only(); + } + Err(anyhow!("Camera hardware encoder error: {}", e)) + } + } + } + either::Right(mut encoder) => { + info!( + "Camera segment encoder started (software): {}x{} -> {}x{} @ {}fps", + video_config.width, + video_config.height, + output_width, + output_height, + frame_rate + ); + + let mut first_timestamp: Option = None; + + while let Ok(Some((frame, timestamp))) = video_rx.recv() { let relative = if let Some(first) = first_timestamp { timestamp.checked_sub(first).unwrap_or(Duration::ZERO) } else { @@ -467,22 +573,34 @@ impl WindowsSegmentedCameraMuxer { Duration::ZERO }; - let texture = upload_mf_buffer_to_texture(&d3d_device, &frame)?; - Ok(Some((texture, duration_to_timespan(relative)))) - }, - |output_sample| { - let mut output = output_clone.lock().unwrap(); - muxer - .write_sample(&output_sample, &mut output) - .map_err(|e| { - windows::core::Error::new( - windows::core::HRESULT(-1), - format!("WriteSample: {e}"), - ) - }) - }, - ) - .context("run camera encoder for segment") + let ffmpeg_frame = match camera_frame_to_ffmpeg(&frame) { + Ok(f) => f, + Err(e) => { + warn!("Failed to convert camera frame: {e}"); + continue; + } + }; + + let Ok(mut output_guard) = output_clone.lock() else { + continue; + }; + + if let Err(e) = + encoder.queue_frame(ffmpeg_frame, relative, &mut output_guard) + { + warn!("Failed to queue camera frame: {e}"); + } + } + + if let Ok(mut output_guard) = output_clone.lock() + && let Err(e) = encoder.flush(&mut output_guard) + { + warn!("Failed to flush software encoder: {e}"); + } + + Ok(()) + } + } })?; ready_rx diff --git a/crates/recording/src/sources/screen_capture/windows.rs b/crates/recording/src/sources/screen_capture/windows.rs index a62d59e3ce..a73932037b 100644 --- a/crates/recording/src/sources/screen_capture/windows.rs +++ b/crates/recording/src/sources/screen_capture/windows.rs @@ -99,6 +99,8 @@ impl ScreenCaptureConfig { Some(Duration::from_secs_f64(1.0 / self.config.fps as f64)); } + settings.fps = Some(self.config.fps); + // Store the display ID instead of GraphicsCaptureItem to avoid COM threading issues // The GraphicsCaptureItem will be created on the capture thread Ok(( diff --git a/crates/recording/src/studio_recording.rs b/crates/recording/src/studio_recording.rs index 0a5a437ab4..325ee7509e 100644 --- a/crates/recording/src/studio_recording.rs +++ b/crates/recording/src/studio_recording.rs @@ -874,7 +874,7 @@ async fn create_segment_pipeline( start_time, fragmented, #[cfg(windows)] - encoder_preferences, + encoder_preferences.clone(), ) .instrument(error_span!("screen-out")) .await @@ -912,14 +912,20 @@ async fn create_segment_pipeline( OutputPipeline::builder(fragments_dir) .with_video::(camera_feed) .with_timestamps(start_time) - .build::(WindowsSegmentedCameraMuxerConfig::default()) + .build::(WindowsSegmentedCameraMuxerConfig { + encoder_preferences: encoder_preferences.clone(), + ..Default::default() + }) .instrument(error_span!("camera-out")) .await } else { OutputPipeline::builder(dir.join("camera.mp4")) .with_video::(camera_feed) .with_timestamps(start_time) - .build::(WindowsCameraMuxerConfig::default()) + .build::(WindowsCameraMuxerConfig { + encoder_preferences: encoder_preferences.clone(), + ..Default::default() + }) .instrument(error_span!("camera-out")) .await }; diff --git a/crates/recording/tests/hardware_compat.rs b/crates/recording/tests/hardware_compat.rs new file mode 100644 index 0000000000..53d7bdb72b --- /dev/null +++ b/crates/recording/tests/hardware_compat.rs @@ -0,0 +1,930 @@ +#![cfg(target_os = "windows")] + +use std::{collections::HashMap, time::Duration}; + +mod test_utils { + use std::sync::Once; + + static INIT: Once = Once::new(); + + pub fn init_tracing() { + INIT.call_once(|| { + tracing_subscriber::fmt() + .with_env_filter( + tracing_subscriber::EnvFilter::from_default_env() + .add_directive(tracing::Level::DEBUG.into()), + ) + .with_test_writer() + .try_init() + .ok(); + }); + } +} + +#[test] +fn test_software_encoding_always_available() { + test_utils::init_tracing(); + + let libx264 = ffmpeg::encoder::find_by_name("libx264"); + assert!( + libx264.is_some(), + "libx264 software encoder must always be available as ultimate fallback" + ); + + let encoder = libx264.unwrap(); + println!("libx264 encoder available: {}", encoder.description()); +} + +#[test] +fn test_swscale_conversion_works() { + test_utils::init_tracing(); + + let config = cap_frame_converter::ConversionConfig::new( + ffmpeg::format::Pixel::BGRA, + 1920, + 1080, + ffmpeg::format::Pixel::NV12, + 1920, + 1080, + ); + + let result = cap_frame_converter::create_converter_with_details(config); + assert!( + result.is_ok(), + "Frame converter should always succeed (with swscale fallback)" + ); + + let selection = result.unwrap(); + println!( + "Converter backend: {:?}, fallback reason: {:?}", + selection.backend, selection.fallback_reason + ); +} + +#[test] +fn test_system_diagnostics_collection() { + test_utils::init_tracing(); + + let diagnostics = cap_recording::diagnostics::collect_diagnostics(); + + println!("=== System Diagnostics ==="); + + if let Some(ref version) = diagnostics.windows_version { + println!( + "Windows: {} (Build {})", + version.display_name, version.build + ); + println!(" Meets requirements: {}", version.meets_requirements); + println!(" Is Windows 11: {}", version.is_windows_11); + } else { + println!("Windows version: Could not detect"); + } + + if let Some(ref gpu) = diagnostics.gpu_info { + println!( + "GPU: {} ({}) - {} MB VRAM", + gpu.description, gpu.vendor, gpu.dedicated_video_memory_mb + ); + } else { + println!("GPU: No dedicated GPU detected (CPU-only or WARP)"); + } + + println!("Available encoders: {:?}", diagnostics.available_encoders); + println!( + "Graphics Capture supported: {}", + diagnostics.graphics_capture_supported + ); + println!( + "D3D11 Video Processor available: {}", + diagnostics.d3d11_video_processor_available + ); + + assert!( + diagnostics + .available_encoders + .contains(&"libx264".to_string()), + "libx264 must be available" + ); +} + +#[test] +fn test_windows_version_detection() { + test_utils::init_tracing(); + + let version = scap_direct3d::WindowsVersion::detect(); + assert!( + version.is_some(), + "Windows version detection should succeed" + ); + + let version = version.unwrap(); + println!( + "Windows Version: {} (Major: {}, Minor: {}, Build: {})", + version.display_name(), + version.major, + version.minor, + version.build + ); + println!( + "Meets minimum requirements (Windows 10 1903+): {}", + version.meets_minimum_requirements() + ); + println!("Is Windows 11: {}", version.is_windows_11()); + println!( + "Supports border control: {}", + version.supports_border_control() + ); + + let graphics_capture_supported = scap_direct3d::is_supported().unwrap_or(false); + if !version.meets_minimum_requirements() && graphics_capture_supported { + println!( + "Note: GetVersionExW returned version {} but Graphics Capture is supported.", + version.display_name() + ); + println!( + "This is expected - Windows compatibility shims can cause incorrect version reporting." + ); + println!("Feature detection (is_supported) is the reliable method, and it returns true."); + } else if !version.meets_minimum_requirements() { + println!("Warning: Windows version appears to be below requirements."); + println!("If Cap works correctly, this may be a version detection issue."); + } +} + +#[test] +fn test_gpu_detection() { + test_utils::init_tracing(); + + let gpu_info = cap_frame_converter::detect_primary_gpu(); + + if let Some(info) = gpu_info { + println!("=== GPU Information ==="); + println!("Description: {}", info.description); + println!("Vendor: {} (0x{:04X})", info.vendor_name(), info.vendor_id); + println!("Device ID: 0x{:04X}", info.device_id); + println!( + "Dedicated VRAM: {} MB", + info.dedicated_video_memory / (1024 * 1024) + ); + + match info.vendor { + cap_frame_converter::GpuVendor::Nvidia => { + println!(" -> NVIDIA GPU: NVENC encoding expected"); + } + cap_frame_converter::GpuVendor::Amd => { + println!(" -> AMD GPU: AMF encoding expected"); + } + cap_frame_converter::GpuVendor::Intel => { + println!(" -> Intel GPU: QSV encoding expected"); + } + cap_frame_converter::GpuVendor::Qualcomm => { + println!(" -> Qualcomm GPU: Software encoding expected"); + } + cap_frame_converter::GpuVendor::Arm => { + println!(" -> ARM GPU: Software encoding expected"); + } + cap_frame_converter::GpuVendor::Microsoft => { + println!(" -> Microsoft WARP: Software rendering/encoding"); + } + cap_frame_converter::GpuVendor::Unknown(id) => { + println!(" -> Unknown GPU vendor (0x{id:04X}): Software fallback"); + } + } + } else { + println!("No GPU detected - system will use software rendering and encoding"); + } +} + +#[test] +fn test_graphics_capture_support() { + test_utils::init_tracing(); + + let supported = scap_direct3d::is_supported().unwrap_or(false); + println!("Windows Graphics Capture API supported: {supported}"); + + if !supported { + let version = scap_direct3d::WindowsVersion::detect(); + if let Some(v) = version { + if !v.meets_minimum_requirements() { + println!( + " -> Reason: Windows version {} does not meet requirements (need 10.0.18362+)", + v.display_name() + ); + } else { + println!(" -> Reason: Graphics Capture may be disabled by group policy"); + } + } + } +} + +#[test] +fn test_camera_enumeration() { + test_utils::init_tracing(); + + let cameras: Vec = cap_camera::list_cameras().collect(); + + println!("=== Camera Enumeration ==="); + println!("Found {} camera(s)", cameras.len()); + + for (i, camera) in cameras.iter().enumerate() { + println!("\n--- Camera {} ---", i + 1); + println!("Display name: {}", camera.display_name()); + println!("Device ID: {}", camera.device_id()); + + if let Some(model_id) = camera.model_id() { + println!("Model ID: {model_id}"); + } + + if let Some(formats) = camera.formats() { + println!("Supported formats: {} format(s)", formats.len()); + + let mut format_summary: HashMap> = HashMap::new(); + for format in &formats { + let key = format!("{}x{}", format.width(), format.height()); + format_summary.entry(key).or_default().push(( + format.width(), + format.height(), + format.frame_rate(), + )); + } + + for (resolution, frame_rates) in format_summary.iter() { + let rates: Vec = frame_rates + .iter() + .map(|(_, _, fps)| format!("{fps:.1}fps")) + .collect(); + println!(" {}: {}", resolution, rates.join(", ")); + } + } else { + println!(" Could not enumerate formats"); + } + } + + if cameras.is_empty() { + println!("\nNo cameras found. This is acceptable for headless/VM environments."); + } +} + +#[test] +fn test_encoder_availability_matrix() { + test_utils::init_tracing(); + + let h264_encoders = [ + ("h264_nvenc", "NVIDIA NVENC"), + ("h264_qsv", "Intel Quick Sync"), + ("h264_amf", "AMD AMF"), + ("h264_mf", "Media Foundation"), + ("libx264", "x264 Software"), + ]; + + let hevc_encoders = [ + ("hevc_nvenc", "NVIDIA NVENC HEVC"), + ("hevc_qsv", "Intel Quick Sync HEVC"), + ("hevc_amf", "AMD AMF HEVC"), + ("hevc_mf", "Media Foundation HEVC"), + ("libx265", "x265 Software"), + ]; + + println!("=== H.264 Encoder Availability ==="); + for (name, description) in h264_encoders { + let available = ffmpeg::encoder::find_by_name(name).is_some(); + let status = if available { "āœ“" } else { "āœ—" }; + println!(" {status} {description} ({name})"); + } + + println!("\n=== HEVC/H.265 Encoder Availability ==="); + for (name, description) in hevc_encoders { + let available = ffmpeg::encoder::find_by_name(name).is_some(); + let status = if available { "āœ“" } else { "āœ—" }; + println!(" {status} {description} ({name})"); + } + + let gpu = cap_frame_converter::detect_primary_gpu(); + println!("\n=== Recommended Encoder Priority ==="); + match gpu.map(|g| g.vendor) { + Some(cap_frame_converter::GpuVendor::Nvidia) => { + println!(" NVIDIA detected: h264_nvenc -> h264_mf -> h264_qsv -> h264_amf -> libx264"); + } + Some(cap_frame_converter::GpuVendor::Amd) => { + println!(" AMD detected: h264_amf -> h264_mf -> h264_nvenc -> h264_qsv -> libx264"); + } + Some(cap_frame_converter::GpuVendor::Intel) => { + println!(" Intel detected: h264_qsv -> h264_mf -> h264_nvenc -> h264_amf -> libx264"); + } + _ => { + println!(" Default: h264_nvenc -> h264_qsv -> h264_amf -> h264_mf -> libx264"); + } + } +} + +#[test] +fn test_d3d11_converter_capability() { + test_utils::init_tracing(); + + let test_configs = [ + ( + "BGRA -> NV12 (1080p)", + ffmpeg::format::Pixel::BGRA, + ffmpeg::format::Pixel::NV12, + 1920, + 1080, + ), + ( + "RGBA -> NV12 (1080p)", + ffmpeg::format::Pixel::RGBA, + ffmpeg::format::Pixel::NV12, + 1920, + 1080, + ), + ( + "BGRA -> NV12 (4K)", + ffmpeg::format::Pixel::BGRA, + ffmpeg::format::Pixel::NV12, + 3840, + 2160, + ), + ( + "YUYV422 -> NV12 (720p)", + ffmpeg::format::Pixel::YUYV422, + ffmpeg::format::Pixel::NV12, + 1280, + 720, + ), + ( + "NV12 -> NV12 (passthrough)", + ffmpeg::format::Pixel::NV12, + ffmpeg::format::Pixel::NV12, + 1920, + 1080, + ), + ]; + + println!("=== D3D11 Converter Capability Tests ==="); + + for (name, input, output, width, height) in test_configs { + let config = + cap_frame_converter::ConversionConfig::new(input, width, height, output, width, height); + + match cap_frame_converter::create_converter_with_details(config) { + Ok(result) => { + let hw = if result.converter.is_hardware_accelerated() { + "GPU" + } else { + "CPU" + }; + println!(" āœ“ {}: {} ({:?})", name, hw, result.backend); + if let Some(reason) = result.fallback_reason { + println!(" Fallback: {reason}"); + } + } + Err(e) => { + println!(" āœ— {name}: Failed - {e}"); + } + } + } +} + +#[test] +fn test_supported_pixel_formats() { + test_utils::init_tracing(); + + let formats = [ + (ffmpeg::format::Pixel::NV12, "NV12"), + (ffmpeg::format::Pixel::YUYV422, "YUYV422"), + (ffmpeg::format::Pixel::BGRA, "BGRA"), + (ffmpeg::format::Pixel::RGBA, "RGBA"), + (ffmpeg::format::Pixel::P010LE, "P010LE (10-bit HDR)"), + (ffmpeg::format::Pixel::YUV420P, "YUV420P"), + (ffmpeg::format::Pixel::RGB24, "RGB24"), + ]; + + println!("=== D3D11 Pixel Format Support ==="); + for (format, name) in formats { + let supported = cap_frame_converter::is_format_supported(format); + let status = if supported { "āœ“" } else { "āœ—" }; + println!(" {status} {name}"); + } +} + +#[test] +#[ignore = "Requires NVIDIA GPU - run with --ignored on NVIDIA systems"] +fn test_nvidia_nvenc_encoding() { + test_utils::init_tracing(); + + let gpu = cap_frame_converter::detect_primary_gpu(); + if !matches!( + gpu.map(|g| g.vendor), + Some(cap_frame_converter::GpuVendor::Nvidia) + ) { + println!("Skipping: No NVIDIA GPU detected"); + return; + } + + let nvenc = ffmpeg::encoder::find_by_name("h264_nvenc"); + assert!( + nvenc.is_some(), + "h264_nvenc should be available on NVIDIA systems" + ); + + let nvenc_hevc = ffmpeg::encoder::find_by_name("hevc_nvenc"); + println!( + "NVIDIA NVENC: H.264={}, HEVC={}", + nvenc.is_some(), + nvenc_hevc.is_some() + ); + + let gpu = gpu.unwrap(); + println!( + "GPU: {} (VRAM: {} MB)", + gpu.description, + gpu.dedicated_video_memory / (1024 * 1024) + ); +} + +#[test] +#[ignore = "Requires AMD GPU - run with --ignored on AMD systems"] +fn test_amd_amf_encoding() { + test_utils::init_tracing(); + + let gpu = cap_frame_converter::detect_primary_gpu(); + if !matches!( + gpu.map(|g| g.vendor), + Some(cap_frame_converter::GpuVendor::Amd) + ) { + println!("Skipping: No AMD GPU detected"); + return; + } + + let amf = ffmpeg::encoder::find_by_name("h264_amf"); + assert!(amf.is_some(), "h264_amf should be available on AMD systems"); + + let amf_hevc = ffmpeg::encoder::find_by_name("hevc_amf"); + println!( + "AMD AMF: H.264={}, HEVC={}", + amf.is_some(), + amf_hevc.is_some() + ); + + let gpu = gpu.unwrap(); + println!( + "GPU: {} (VRAM: {} MB)", + gpu.description, + gpu.dedicated_video_memory / (1024 * 1024) + ); +} + +#[test] +#[ignore = "Requires Intel GPU - run with --ignored on Intel systems"] +fn test_intel_qsv_encoding() { + test_utils::init_tracing(); + + let gpu = cap_frame_converter::detect_primary_gpu(); + if !matches!( + gpu.map(|g| g.vendor), + Some(cap_frame_converter::GpuVendor::Intel) + ) { + println!("Skipping: No Intel GPU detected"); + return; + } + + let qsv = ffmpeg::encoder::find_by_name("h264_qsv"); + assert!( + qsv.is_some(), + "h264_qsv should be available on Intel systems" + ); + + let qsv_hevc = ffmpeg::encoder::find_by_name("hevc_qsv"); + println!( + "Intel Quick Sync: H.264={}, HEVC={}", + qsv.is_some(), + qsv_hevc.is_some() + ); + + let gpu = gpu.unwrap(); + println!( + "GPU: {} (VRAM: {} MB)", + gpu.description, + gpu.dedicated_video_memory / (1024 * 1024) + ); +} + +#[test] +#[ignore = "Requires actual camera - run with --ignored when camera is connected"] +fn test_camera_capture_basic() { + test_utils::init_tracing(); + + let cameras: Vec = cap_camera::list_cameras().collect(); + if cameras.is_empty() { + println!("No cameras available for capture test"); + return; + } + + let camera = &cameras[0]; + println!("Testing camera: {}", camera.display_name()); + + let formats = camera.formats(); + if formats.is_none() { + println!("Could not get camera formats"); + return; + } + + let formats = formats.unwrap(); + if formats.is_empty() { + println!("Camera has no supported formats"); + return; + } + + let format = formats + .iter() + .find(|f| f.width() == 1280 && f.height() == 720) + .or_else(|| formats.first()) + .cloned() + .unwrap(); + + println!( + "Using format: {}x{} @ {}fps", + format.width(), + format.height(), + format.frame_rate() + ); + + let frame_count = std::sync::Arc::new(std::sync::atomic::AtomicU32::new(0)); + let frame_count_clone = frame_count.clone(); + + let handle = camera.start_capturing(format, move |_frame| { + frame_count_clone.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + }); + + match handle { + Ok(handle) => { + std::thread::sleep(Duration::from_secs(2)); + + let frames = frame_count.load(std::sync::atomic::Ordering::Relaxed); + println!("Captured {frames} frames in 2 seconds"); + + let _ = handle.stop_capturing(); + + assert!(frames > 0, "Should have captured at least one frame"); + } + Err(e) => { + println!("Failed to start capture: {e:?}"); + } + } +} + +#[test] +#[ignore = "Requires virtual camera (OBS Virtual Camera) - run with --ignored when available"] +fn test_virtual_camera_detection() { + test_utils::init_tracing(); + + let cameras: Vec = cap_camera::list_cameras().collect(); + + let virtual_camera_keywords = ["obs", "virtual", "snap", "manycam", "xsplit", "droidcam"]; + + println!("=== Virtual Camera Detection ==="); + for camera in &cameras { + let name_lower = camera.display_name().to_lowercase(); + let is_virtual = virtual_camera_keywords + .iter() + .any(|keyword| name_lower.contains(keyword)); + + if is_virtual { + println!(" [VIRTUAL] {}", camera.display_name()); + } else { + println!(" [PHYSICAL] {}", camera.display_name()); + } + } + + let virtual_count = cameras + .iter() + .filter(|c| { + let name = c.display_name().to_lowercase(); + virtual_camera_keywords + .iter() + .any(|keyword| name.contains(keyword)) + }) + .count(); + + println!( + "\nFound {} virtual camera(s), {} physical camera(s)", + virtual_count, + cameras.len() - virtual_count + ); +} + +#[test] +#[ignore = "Requires capture card (Elgato, etc.) - run with --ignored when available"] +fn test_capture_card_detection() { + test_utils::init_tracing(); + + let cameras: Vec = cap_camera::list_cameras().collect(); + + let capture_card_keywords = [ + "elgato", + "avermedia", + "magewell", + "blackmagic", + "decklink", + "cam link", + "hd60", + "4k60", + ]; + + println!("=== Capture Card Detection ==="); + for camera in &cameras { + let name_lower = camera.display_name().to_lowercase(); + let is_capture_card = capture_card_keywords + .iter() + .any(|keyword| name_lower.contains(keyword)); + + if is_capture_card { + println!(" [CAPTURE CARD] {}", camera.display_name()); + + if let Some(formats) = camera.formats() { + let max_res = formats.iter().max_by_key(|f| f.width() * f.height()); + if let Some(max) = max_res { + println!( + " Max resolution: {}x{} @ {}fps", + max.width(), + max.height(), + max.frame_rate() + ); + } + } + } + } +} + +#[test] +fn test_hardware_compatibility_summary() { + test_utils::init_tracing(); + + println!("\nā•”ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•—"); + println!("ā•‘ HARDWARE COMPATIBILITY SUMMARY ā•‘"); + println!("ā• ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•£"); + + let version = scap_direct3d::WindowsVersion::detect(); + let gpu = cap_frame_converter::detect_primary_gpu(); + let diagnostics = cap_recording::diagnostics::collect_diagnostics(); + + let windows_status = if diagnostics.graphics_capture_supported { + if let Some(v) = &version { + format!("āœ“ {} (Graphics Capture OK)", v.display_name()) + } else { + "āœ“ Graphics Capture supported".to_string() + } + } else if let Some(v) = &version { + format!("āœ— {} - Graphics Capture unavailable", v.display_name()) + } else { + "? Unknown".to_string() + }; + println!("ā•‘ Windows: {windows_status:<52} ā•‘"); + + let gpu_status = if let Some(g) = gpu { + format!( + "āœ“ {} ({} MB)", + truncate_string(&g.description, 35), + g.dedicated_video_memory / (1024 * 1024) + ) + } else { + "āš  No GPU (WARP software rendering)".to_string() + }; + println!("ā•‘ GPU: {gpu_status:<56} ā•‘"); + + let capture_status = if diagnostics.graphics_capture_supported { + "āœ“ Available" + } else { + "āœ— Unavailable" + }; + println!("ā•‘ Screen Capture: {capture_status:<45} ā•‘"); + + let d3d11_status = if diagnostics.d3d11_video_processor_available { + "āœ“ GPU accelerated" + } else { + "āš  CPU fallback (swscale)" + }; + println!("ā•‘ Frame Conversion: {d3d11_status:<43} ā•‘"); + + let hw_encoders: Vec<&str> = diagnostics + .available_encoders + .iter() + .filter(|e| !e.starts_with("lib")) + .map(|s| s.as_str()) + .collect(); + let encoder_status = if !hw_encoders.is_empty() { + format!("āœ“ {} hardware encoder(s)", hw_encoders.len()) + } else { + "āš  Software only (libx264)".to_string() + }; + println!("ā•‘ Encoding: {encoder_status:<51} ā•‘"); + + let cameras: Vec = cap_camera::list_cameras().collect(); + let camera_status = format!("{} camera(s) detected", cameras.len()); + println!("ā•‘ Cameras: {camera_status:<52} ā•‘"); + + println!("ā• ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•£"); + + let all_good = diagnostics.graphics_capture_supported + && diagnostics + .available_encoders + .contains(&"libx264".to_string()); + + if all_good { + println!("ā•‘ Status: āœ“ SYSTEM COMPATIBLE ā•‘"); + } else { + println!("ā•‘ Status: āš  COMPATIBILITY ISSUES DETECTED ā•‘"); + } + println!("ā•šā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•\n"); +} + +fn truncate_string(s: &str, max_len: usize) -> String { + if s.chars().count() <= max_len { + s.to_string() + } else { + let truncate_at = max_len.saturating_sub(3); + let truncated: String = s.chars().take(truncate_at).collect(); + format!("{truncated}...") + } +} + +#[test] +#[ignore = "Intensive test - run with --ignored for full validation"] +fn test_frame_conversion_performance() { + test_utils::init_tracing(); + + let config = cap_frame_converter::ConversionConfig::new( + ffmpeg::format::Pixel::BGRA, + 1920, + 1080, + ffmpeg::format::Pixel::NV12, + 1920, + 1080, + ); + + let result = cap_frame_converter::create_converter_with_details(config.clone()); + if result.is_err() { + println!("Could not create converter: {:?}", result.err()); + return; + } + + let selection = result.unwrap(); + println!( + "Testing converter: {:?} (hardware: {})", + selection.backend, + selection.converter.is_hardware_accelerated() + ); + + let test_frame = ffmpeg::frame::Video::new(ffmpeg::format::Pixel::BGRA, 1920, 1080); + + let warmup_iterations = 10; + let test_iterations = 100; + + for _ in 0..warmup_iterations { + let frame = test_frame.clone(); + let _ = selection.converter.convert(frame); + } + + let start = std::time::Instant::now(); + for _ in 0..test_iterations { + let frame = test_frame.clone(); + let _ = selection.converter.convert(frame); + } + let elapsed = start.elapsed(); + + let avg_ms = elapsed.as_secs_f64() * 1000.0 / test_iterations as f64; + let fps_capacity = 1000.0 / avg_ms; + + println!("Performance: {avg_ms:.2}ms/frame avg ({fps_capacity:.1} fps capacity)"); + + let target_fps = 60.0; + let target_ms = 1000.0 / target_fps; + if avg_ms < target_ms { + println!("āœ“ Can sustain {}fps recording", target_fps as u32); + } else { + println!( + "āš  May struggle with {}fps (need {:.2}ms, got {:.2}ms)", + target_fps as u32, target_ms, avg_ms + ); + } +} + +#[test] +fn test_multi_gpu_detection() { + test_utils::init_tracing(); + + println!("=== Multi-GPU Detection ==="); + println!("Primary GPU detection uses DXGI EnumAdapters(0)"); + + if let Some(gpu) = cap_frame_converter::detect_primary_gpu() { + println!("Primary adapter: {}", gpu.description); + println!("Vendor: {} (0x{:04X})", gpu.vendor_name(), gpu.vendor_id); + + if gpu.dedicated_video_memory < 512 * 1024 * 1024 { + println!("āš  Low VRAM (<512MB) - software encoding recommended"); + } else if gpu.dedicated_video_memory < 2 * 1024 * 1024 * 1024 { + println!("āœ“ Adequate VRAM for 1080p recording"); + } else { + println!("āœ“ Ample VRAM for 4K recording"); + } + } else { + println!("No dedicated GPU found - using integrated or software rendering"); + } +} + +#[test] +fn test_minimum_requirements_check() { + test_utils::init_tracing(); + + let mut requirements_met = true; + let mut warnings = Vec::new(); + + println!("=== Minimum Requirements Check ===\n"); + + println!("Required:"); + + let graphics_capture_supported = scap_direct3d::is_supported().unwrap_or(false); + if graphics_capture_supported { + println!(" āœ“ Windows Graphics Capture API"); + } else { + println!(" āœ— Windows Graphics Capture API unavailable"); + requirements_met = false; + } + + let version = scap_direct3d::WindowsVersion::detect(); + if let Some(v) = &version { + if v.meets_minimum_requirements() { + println!( + " āœ“ Windows 10 version 1903 or later (reported: {})", + v.display_name() + ); + } else if graphics_capture_supported { + println!( + " āš  Windows version reported as {} (may be inaccurate due to compat shims)", + v.display_name() + ); + println!(" Graphics Capture works, so actual version is sufficient."); + } else { + println!( + " āœ— Windows version {} is below minimum (need 10.0.18362+)", + v.display_name() + ); + requirements_met = false; + } + } else { + println!(" ? Could not detect Windows version"); + if !graphics_capture_supported { + requirements_met = false; + } + } + + if ffmpeg::encoder::find_by_name("libx264").is_some() { + println!(" āœ“ FFmpeg with libx264 encoder"); + } else { + println!(" āœ— libx264 encoder not available"); + requirements_met = false; + } + + println!("\nRecommended:"); + if cap_frame_converter::detect_primary_gpu().is_some() { + println!(" āœ“ Dedicated or integrated GPU"); + } else { + println!(" āš  No GPU detected (will use software rendering)"); + warnings.push("Performance may be reduced without GPU acceleration"); + } + + let diagnostics = cap_recording::diagnostics::collect_diagnostics(); + let hw_encoders: Vec<&str> = diagnostics + .available_encoders + .iter() + .filter(|e| !e.starts_with("lib")) + .map(|s| s.as_str()) + .collect(); + + if !hw_encoders.is_empty() { + println!(" āœ“ Hardware video encoder ({hw_encoders:?})"); + } else { + println!(" āš  No hardware encoders (will use CPU encoding)"); + warnings.push("CPU encoding may impact system performance"); + } + + if diagnostics.d3d11_video_processor_available { + println!(" āœ“ D3D11 video processor for frame conversion"); + } else { + println!(" āš  D3D11 video processor unavailable (will use CPU conversion)"); + warnings.push("CPU frame conversion may impact performance"); + } + + println!("\n=== Result ==="); + if requirements_met && warnings.is_empty() { + println!("āœ“ All requirements met - system fully compatible"); + } else if requirements_met { + println!("āš  Requirements met with warnings:"); + for warning in &warnings { + println!(" - {warning}"); + } + } else { + println!("āœ— Missing required components - Cap may not function correctly"); + } + + assert!(requirements_met, "Minimum requirements not met"); +} diff --git a/crates/rendering/src/cpu_yuv.rs b/crates/rendering/src/cpu_yuv.rs index 7dbb0a26bb..ea929e5358 100644 --- a/crates/rendering/src/cpu_yuv.rs +++ b/crates/rendering/src/cpu_yuv.rs @@ -990,11 +990,7 @@ mod tests { let diff = (*s as i32 - *d as i32).abs(); assert!( diff <= 2, - "Mismatch at index {}: scalar={}, simd={}, diff={}", - i, - s, - d, - diff + "Mismatch at index {i}: scalar={s}, simd={d}, diff={diff}" ); } } @@ -1064,11 +1060,7 @@ mod tests { let diff = (*a as i32 - *b as i32).abs(); assert!( diff <= 2, - "Mismatch at index {}: expected={}, got={}, diff={}", - i, - a, - b, - diff + "Mismatch at index {i}: expected={a}, got={b}, diff={diff}" ); } } @@ -1119,11 +1111,7 @@ mod tests { let diff = (*s as i32 - *d as i32).abs(); assert!( diff <= 2, - "YUV420P mismatch at index {}: scalar={}, simd={}, diff={}", - i, - s, - d, - diff + "YUV420P mismatch at index {i}: scalar={s}, simd={d}, diff={diff}" ); } } diff --git a/crates/rendering/src/decoder/avassetreader.rs b/crates/rendering/src/decoder/avassetreader.rs index 8192e0c2ce..408490097b 100644 --- a/crates/rendering/src/decoder/avassetreader.rs +++ b/crates/rendering/src/decoder/avassetreader.rs @@ -18,7 +18,7 @@ use tokio::{runtime::Handle as TokioHandle, sync::oneshot}; use crate::{DecodedFrame, PixelFormat}; use super::frame_converter::{copy_bgra_to_rgba, copy_rgba_plane}; -use super::{FRAME_CACHE_SIZE, VideoDecoderMessage, pts_to_frame}; +use super::{DecoderInitResult, DecoderType, FRAME_CACHE_SIZE, VideoDecoderMessage, pts_to_frame}; #[derive(Clone)] struct ProcessedFrame { @@ -253,7 +253,7 @@ impl AVAssetReaderDecoder { path: PathBuf, fps: u32, rx: mpsc::Receiver, - ready_tx: oneshot::Sender>, + ready_tx: oneshot::Sender>, ) { let handle = tokio::runtime::Handle::current(); @@ -265,14 +265,11 @@ impl AVAssetReaderDecoder { path: PathBuf, fps: u32, rx: mpsc::Receiver, - ready_tx: oneshot::Sender>, + ready_tx: oneshot::Sender>, tokio_handle: tokio::runtime::Handle, ) { let mut this = match AVAssetReaderDecoder::new(path, tokio_handle) { - Ok(v) => { - ready_tx.send(Ok(())).ok(); - v - } + Ok(v) => v, Err(e) => { ready_tx.send(Err(e)).ok(); return; @@ -282,6 +279,13 @@ impl AVAssetReaderDecoder { let video_width = this.inner.width(); let video_height = this.inner.height(); + let init_result = DecoderInitResult { + width: video_width, + height: video_height, + decoder_type: DecoderType::AVAssetReader, + }; + ready_tx.send(Ok(init_result)).ok(); + let mut cache = BTreeMap::::new(); #[allow(unused)] diff --git a/crates/rendering/src/decoder/ffmpeg.rs b/crates/rendering/src/decoder/ffmpeg.rs index 450e8db077..961e4cec28 100644 --- a/crates/rendering/src/decoder/ffmpeg.rs +++ b/crates/rendering/src/decoder/ffmpeg.rs @@ -10,10 +10,14 @@ use std::{ sync::{Arc, mpsc}, }; use tokio::sync::oneshot; +use tracing::info; use crate::{DecodedFrame, PixelFormat}; -use super::{FRAME_CACHE_SIZE, VideoDecoderMessage, frame_converter::FrameConverter, pts_to_frame}; +use super::{ + DecoderInitResult, DecoderType, FRAME_CACHE_SIZE, VideoDecoderMessage, + frame_converter::FrameConverter, pts_to_frame, +}; #[derive(Clone)] struct ProcessedFrame { @@ -134,29 +138,40 @@ pub struct FfmpegDecoder; impl FfmpegDecoder { pub fn spawn( - _name: &'static str, + name: &'static str, path: PathBuf, fps: u32, rx: mpsc::Receiver, - ready_tx: oneshot::Sender>, + ready_tx: oneshot::Sender>, ) -> Result<(), String> { - let (continue_tx, continue_rx) = mpsc::channel(); + let (continue_tx, continue_rx) = mpsc::channel::>(); std::thread::spawn(move || { - let mut this = match cap_video_decode::FFmpegDecoder::new( - path, - Some(if cfg!(target_os = "macos") { - AVHWDeviceType::AV_HWDEVICE_TYPE_VIDEOTOOLBOX - } else { - AVHWDeviceType::AV_HWDEVICE_TYPE_DXVA2 - }), - ) { + let hw_device_type = if cfg!(target_os = "macos") { + Some(AVHWDeviceType::AV_HWDEVICE_TYPE_VIDEOTOOLBOX) + } else if cfg!(target_os = "linux") { + Some(AVHWDeviceType::AV_HWDEVICE_TYPE_VAAPI) + } else if cfg!(target_os = "windows") { + Some(AVHWDeviceType::AV_HWDEVICE_TYPE_DXVA2) + } else { + None + }; + + let mut this = match cap_video_decode::FFmpegDecoder::new(path.clone(), hw_device_type) + { Err(e) => { let _ = continue_tx.send(Err(e)); return; } Ok(v) => { - let _ = continue_tx.send(Ok(())); + let is_hw = v.is_hardware_accelerated(); + let width = v.decoder().width(); + let height = v.decoder().height(); + info!( + "FFmpeg decoder created for '{}': {}x{}, hw_accel={}", + name, width, height, is_hw + ); + let _ = continue_tx.send(Ok((width, height, is_hw))); v } }; @@ -165,10 +180,9 @@ impl FfmpegDecoder { let start_time = this.start_time(); let video_width = this.decoder().width(); let video_height = this.decoder().height(); + let is_hw = this.is_hardware_accelerated(); let mut cache = BTreeMap::::new(); - // active frame is a frame that triggered decode. - // frames that are within render_more_margin of this frame won't trigger decode. #[allow(unused)] let mut last_active_frame = None::; @@ -177,7 +191,17 @@ impl FfmpegDecoder { let mut frames = this.frames(); let mut converter = FrameConverter::new(); - let _ = ready_tx.send(Ok(())); + let decoder_type = if is_hw { + DecoderType::FFmpegHardware + } else { + DecoderType::FFmpegSoftware + }; + let init_result = DecoderInitResult { + width: video_width, + height: video_height, + decoder_type, + }; + let _ = ready_tx.send(Ok(init_result)); while let Ok(r) = rx.recv() { match r { @@ -356,9 +380,7 @@ impl FfmpegDecoder { } }); - continue_rx.recv().map_err(|e| e.to_string())??; - - Ok(()) + continue_rx.recv().map_err(|e| e.to_string())?.map(|_| ()) } } diff --git a/crates/rendering/src/decoder/media_foundation.rs b/crates/rendering/src/decoder/media_foundation.rs index 2963f09c04..18a142b734 100644 --- a/crates/rendering/src/decoder/media_foundation.rs +++ b/crates/rendering/src/decoder/media_foundation.rs @@ -2,12 +2,88 @@ use std::{ collections::BTreeMap, path::PathBuf, sync::{Arc, mpsc}, + time::{Duration, Instant}, }; use tokio::sync::oneshot; use tracing::{debug, info, warn}; use windows::Win32::{Foundation::HANDLE, Graphics::Direct3D11::ID3D11Texture2D}; -use super::{DecodedFrame, FRAME_CACHE_SIZE, VideoDecoderMessage}; +use super::{DecodedFrame, DecoderInitResult, DecoderType, FRAME_CACHE_SIZE, VideoDecoderMessage}; + +struct DecoderHealthMonitor { + consecutive_errors: u32, + consecutive_texture_read_failures: u32, + total_frames_decoded: u64, + total_errors: u64, + last_successful_decode: Instant, + frame_decode_times: [Duration; 32], + frame_decode_index: usize, + slow_frame_count: u32, +} + +impl DecoderHealthMonitor { + fn new() -> Self { + Self { + consecutive_errors: 0, + consecutive_texture_read_failures: 0, + total_frames_decoded: 0, + total_errors: 0, + last_successful_decode: Instant::now(), + frame_decode_times: [Duration::ZERO; 32], + frame_decode_index: 0, + slow_frame_count: 0, + } + } + + fn record_success(&mut self, decode_time: Duration) { + self.consecutive_errors = 0; + self.consecutive_texture_read_failures = 0; + self.total_frames_decoded += 1; + self.last_successful_decode = Instant::now(); + + self.frame_decode_times[self.frame_decode_index] = decode_time; + self.frame_decode_index = (self.frame_decode_index + 1) % 32; + + const SLOW_FRAME_THRESHOLD: Duration = Duration::from_millis(100); + if decode_time > SLOW_FRAME_THRESHOLD { + self.slow_frame_count += 1; + } + } + + fn record_error(&mut self) { + self.consecutive_errors += 1; + self.total_errors += 1; + } + + fn record_texture_read_failure(&mut self) { + self.consecutive_texture_read_failures += 1; + } + + fn is_healthy(&self) -> bool { + const MAX_CONSECUTIVE_ERRORS: u32 = 10; + const MAX_CONSECUTIVE_TEXTURE_FAILURES: u32 = 5; + const MAX_TIME_SINCE_SUCCESS: Duration = Duration::from_secs(5); + + self.consecutive_errors < MAX_CONSECUTIVE_ERRORS + && self.consecutive_texture_read_failures < MAX_CONSECUTIVE_TEXTURE_FAILURES + && self.last_successful_decode.elapsed() < MAX_TIME_SINCE_SUCCESS + } + + #[allow(dead_code)] + fn average_decode_time(&self) -> Duration { + let sum: Duration = self.frame_decode_times.iter().sum(); + sum / 32 + } + + #[allow(dead_code)] + fn get_health_summary(&self) -> (u64, u64, u32) { + ( + self.total_frames_decoded, + self.total_errors, + self.slow_frame_count, + ) + } +} #[derive(Clone)] struct CachedFrame { @@ -49,7 +125,7 @@ impl MFDecoder { path: PathBuf, fps: u32, rx: mpsc::Receiver, - ready_tx: oneshot::Sender>, + ready_tx: oneshot::Sender>, ) -> Result<(), String> { let (continue_tx, continue_rx) = mpsc::channel(); @@ -60,14 +136,34 @@ impl MFDecoder { return; } Ok(v) => { + let width = v.width(); + let height = v.height(); + let caps = v.capabilities(); + + let exceeds_hw_limits = width > caps.max_width || height > caps.max_height; + + if exceeds_hw_limits { + warn!( + "Video '{}' dimensions {}x{} exceed hardware decoder limits ({}x{})", + name, width, height, caps.max_width, caps.max_height + ); + let _ = continue_tx.send(Err(format!( + "Video dimensions {}x{} exceed hardware decoder limits {}x{}", + width, height, caps.max_width, caps.max_height + ))); + return; + } + info!( - "MediaFoundation decoder created for '{}': {}x{} @ {:?}fps", + "MediaFoundation decoder created for '{}': {}x{} @ {:?}fps (hw max: {}x{})", name, - v.width(), - v.height(), - v.frame_rate() + width, + height, + v.frame_rate(), + caps.max_width, + caps.max_height ); - let _ = continue_tx.send(Ok(())); + let _ = continue_tx.send(Ok((width, height))); v } }; @@ -77,8 +173,14 @@ impl MFDecoder { let mut cache = BTreeMap::::new(); let mut last_decoded_frame: Option = None; + let mut health = DecoderHealthMonitor::new(); - let _ = ready_tx.send(Ok(())); + let init_result = DecoderInitResult { + width: video_width, + height: video_height, + decoder_type: DecoderType::MediaFoundation, + }; + let _ = ready_tx.send(Ok(init_result)); while let Ok(r) = rx.recv() { match r { @@ -87,6 +189,16 @@ impl MFDecoder { continue; } + if !health.is_healthy() { + warn!( + name = name, + consecutive_errors = health.consecutive_errors, + texture_failures = health.consecutive_texture_read_failures, + total_decoded = health.total_frames_decoded, + "MediaFoundation decoder unhealthy, performance may degrade" + ); + } + let requested_frame = (requested_time * fps as f32).floor() as u32; if let Some(cached) = cache.get(&requested_frame) { @@ -123,8 +235,10 @@ impl MFDecoder { let mut last_valid_frame: Option = None; loop { + let decode_start = Instant::now(); match decoder.read_sample() { Ok(Some(mf_frame)) => { + let decode_time = decode_start.elapsed(); let frame_number = pts_100ns_to_frame(mf_frame.pts, fps); let nv12_data = match decoder.read_texture_to_cpu( @@ -133,6 +247,7 @@ impl MFDecoder { mf_frame.height, ) { Ok(data) => { + health.record_success(decode_time); debug!( frame = frame_number, data_len = data.data.len(), @@ -140,11 +255,13 @@ impl MFDecoder { uv_stride = data.uv_stride, width = mf_frame.width, height = mf_frame.height, + decode_ms = decode_time.as_millis(), "read_texture_to_cpu succeeded" ); Some(Arc::new(data)) } Err(e) => { + health.record_texture_read_failure(); warn!( "Failed to read texture to CPU for frame {frame_number}: {e}" ); @@ -211,7 +328,11 @@ impl MFDecoder { break; } Err(e) => { - warn!("MediaFoundation read_sample error: {e}"); + health.record_error(); + warn!( + consecutive_errors = health.consecutive_errors, + "MediaFoundation read_sample error: {e}" + ); break; } } @@ -243,9 +364,7 @@ impl MFDecoder { } }); - continue_rx.recv().map_err(|e| e.to_string())??; - - Ok(()) + continue_rx.recv().map_err(|e| e.to_string())?.map(|_| ()) } } diff --git a/crates/rendering/src/decoder/mod.rs b/crates/rendering/src/decoder/mod.rs index 765f0d45ad..bad0ddbaf1 100644 --- a/crates/rendering/src/decoder/mod.rs +++ b/crates/rendering/src/decoder/mod.rs @@ -3,9 +3,12 @@ use std::{ fmt, path::PathBuf, sync::{Arc, mpsc}, + time::Duration, }; use tokio::sync::oneshot; -use tracing::debug; +#[cfg(target_os = "windows")] +use tracing::warn; +use tracing::{debug, info}; #[cfg(target_os = "macos")] mod avassetreader; @@ -14,6 +17,57 @@ mod frame_converter; #[cfg(target_os = "windows")] mod media_foundation; +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DecoderType { + #[cfg(target_os = "macos")] + AVAssetReader, + #[cfg(target_os = "windows")] + MediaFoundation, + FFmpegHardware, + FFmpegSoftware, +} + +impl DecoderType { + pub fn is_hardware_accelerated(&self) -> bool { + match self { + #[cfg(target_os = "macos")] + DecoderType::AVAssetReader => true, + #[cfg(target_os = "windows")] + DecoderType::MediaFoundation => true, + DecoderType::FFmpegHardware => true, + DecoderType::FFmpegSoftware => false, + } + } +} + +impl fmt::Display for DecoderType { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + #[cfg(target_os = "macos")] + DecoderType::AVAssetReader => write!(f, "AVAssetReader (hardware)"), + #[cfg(target_os = "windows")] + DecoderType::MediaFoundation => write!(f, "MediaFoundation (hardware)"), + DecoderType::FFmpegHardware => write!(f, "FFmpeg (hardware)"), + DecoderType::FFmpegSoftware => write!(f, "FFmpeg (software)"), + } + } +} + +#[derive(Debug, Clone)] +pub struct DecoderStatus { + pub decoder_type: DecoderType, + pub video_width: u32, + pub video_height: u32, + pub fallback_reason: Option, +} + +#[derive(Debug, Clone)] +pub struct DecoderInitResult { + pub width: u32, + pub height: u32, + pub decoder_type: DecoderType, +} + #[cfg(target_os = "macos")] use cidre::{arc::R, cv}; @@ -403,6 +457,7 @@ pub const FRAME_CACHE_SIZE: usize = 750; pub struct AsyncVideoDecoderHandle { sender: mpsc::Sender, offset: f64, + status: DecoderStatus, } impl AsyncVideoDecoderHandle { @@ -425,6 +480,26 @@ impl AsyncVideoDecoderHandle { pub fn get_time(&self, time: f32) -> f32 { time + self.offset as f32 } + + pub fn decoder_status(&self) -> &DecoderStatus { + &self.status + } + + pub fn decoder_type(&self) -> DecoderType { + self.status.decoder_type + } + + pub fn is_hardware_accelerated(&self) -> bool { + self.status.decoder_type.is_hardware_accelerated() + } + + pub fn video_dimensions(&self) -> (u32, u32) { + (self.status.video_width, self.status.video_height) + } + + pub fn fallback_reason(&self) -> Option<&str> { + self.status.fallback_reason.as_deref() + } } pub async fn spawn_decoder( @@ -433,61 +508,160 @@ pub async fn spawn_decoder( fps: u32, offset: f64, ) -> Result { - let (ready_tx, ready_rx) = oneshot::channel::>(); - let (tx, rx) = mpsc::channel(); - - let handle = AsyncVideoDecoderHandle { sender: tx, offset }; - let path_display = path.display().to_string(); + let timeout_duration = Duration::from_secs(30); #[cfg(target_os = "macos")] { + let (ready_tx, ready_rx) = oneshot::channel::>(); + let (tx, rx) = mpsc::channel(); + avassetreader::AVAssetReaderDecoder::spawn(name, path, fps, rx, ready_tx); + + match tokio::time::timeout(timeout_duration, ready_rx).await { + Ok(Ok(Ok(init_result))) => { + info!( + "Video '{}' using {} decoder ({}x{})", + name, init_result.decoder_type, init_result.width, init_result.height + ); + let status = DecoderStatus { + decoder_type: init_result.decoder_type, + video_width: init_result.width, + video_height: init_result.height, + fallback_reason: None, + }; + Ok(AsyncVideoDecoderHandle { + sender: tx, + offset, + status, + }) + } + Ok(Ok(Err(e))) => Err(format!("'{name}' decoder initialization failed: {e}")), + Ok(Err(e)) => Err(format!("'{name}' decoder channel closed: {e}")), + Err(_) => Err(format!( + "'{name}' decoder timed out after 30s initializing: {path_display}" + )), + } } #[cfg(target_os = "windows")] { + let (ready_tx, ready_rx) = oneshot::channel::>(); + let (tx, rx) = mpsc::channel(); + match media_foundation::MFDecoder::spawn(name, path.clone(), fps, rx, ready_tx) { - Ok(()) => { - debug!("Using MediaFoundation decoder for '{name}'"); - } + Ok(()) => match tokio::time::timeout(timeout_duration, ready_rx).await { + Ok(Ok(Ok(init_result))) => { + info!( + "Video '{}' using {} decoder ({}x{})", + name, init_result.decoder_type, init_result.width, init_result.height + ); + let status = DecoderStatus { + decoder_type: init_result.decoder_type, + video_width: init_result.width, + video_height: init_result.height, + fallback_reason: None, + }; + return Ok(AsyncVideoDecoderHandle { + sender: tx, + offset, + status, + }); + } + Ok(Ok(Err(e))) => { + warn!( + "MediaFoundation decoder ready but failed for '{}': {}, falling back to FFmpeg", + name, e + ); + } + Ok(Err(e)) => { + warn!( + "MediaFoundation decoder channel closed for '{}': {}, falling back to FFmpeg", + name, e + ); + } + Err(_) => { + warn!( + "MediaFoundation decoder timed out for '{}', falling back to FFmpeg", + name + ); + } + }, Err(mf_err) => { debug!( - "MediaFoundation decoder failed for '{name}': {mf_err}, falling back to FFmpeg" + "MediaFoundation decoder spawn failed for '{}': {}, falling back to FFmpeg", + name, mf_err ); - let (ready_tx, ready_rx_new) = oneshot::channel::>(); - let (tx, rx) = mpsc::channel(); - let handle = AsyncVideoDecoderHandle { sender: tx, offset }; - - ffmpeg::FfmpegDecoder::spawn(name, path, fps, rx, ready_tx) - .map_err(|e| format!("'{name}' decoder / {e}"))?; - - return match tokio::time::timeout(std::time::Duration::from_secs(30), ready_rx_new) - .await - { - Ok(result) => result - .map_err(|e| format!("'{name}' decoder channel closed: {e}"))? - .map(|()| handle), - Err(_) => Err(format!( - "'{name}' decoder timed out after 30s initializing: {path_display}" - )), + } + } + + let fallback_reason = + format!("MediaFoundation decoder unavailable for '{name}', using FFmpeg fallback"); + let (ready_tx, ready_rx) = oneshot::channel::>(); + let (tx, rx) = mpsc::channel(); + + ffmpeg::FfmpegDecoder::spawn(name, path, fps, rx, ready_tx) + .map_err(|e| format!("'{name}' FFmpeg fallback decoder / {e}"))?; + + match tokio::time::timeout(timeout_duration, ready_rx).await { + Ok(Ok(Ok(init_result))) => { + info!( + "Video '{}' using {} decoder ({}x{}) [fallback]", + name, init_result.decoder_type, init_result.width, init_result.height + ); + let status = DecoderStatus { + decoder_type: init_result.decoder_type, + video_width: init_result.width, + video_height: init_result.height, + fallback_reason: Some(fallback_reason), }; + Ok(AsyncVideoDecoderHandle { + sender: tx, + offset, + status, + }) } + Ok(Ok(Err(e))) => Err(format!( + "'{name}' FFmpeg decoder initialization failed: {e}" + )), + Ok(Err(e)) => Err(format!("'{name}' FFmpeg decoder channel closed: {e}")), + Err(_) => Err(format!( + "'{name}' FFmpeg decoder timed out after 30s initializing: {path_display}" + )), } } #[cfg(not(any(target_os = "macos", target_os = "windows")))] { + let (ready_tx, ready_rx) = oneshot::channel::>(); + let (tx, rx) = mpsc::channel(); + ffmpeg::FfmpegDecoder::spawn(name, path, fps, rx, ready_tx) .map_err(|e| format!("'{name}' decoder / {e}"))?; - } - match tokio::time::timeout(std::time::Duration::from_secs(30), ready_rx).await { - Ok(result) => result - .map_err(|e| format!("'{name}' decoder channel closed: {e}"))? - .map(|()| handle), - Err(_) => Err(format!( - "'{name}' decoder timed out after 30s initializing: {path_display}" - )), + match tokio::time::timeout(timeout_duration, ready_rx).await { + Ok(Ok(Ok(init_result))) => { + info!( + "Video '{}' using {} decoder ({}x{})", + name, init_result.decoder_type, init_result.width, init_result.height + ); + let status = DecoderStatus { + decoder_type: init_result.decoder_type, + video_width: init_result.width, + video_height: init_result.height, + fallback_reason: None, + }; + Ok(AsyncVideoDecoderHandle { + sender: tx, + offset, + status, + }) + } + Ok(Ok(Err(e))) => Err(format!("'{name}' decoder initialization failed: {e}")), + Ok(Err(e)) => Err(format!("'{name}' decoder channel closed: {e}")), + Err(_) => Err(format!( + "'{name}' decoder timed out after 30s initializing: {path_display}" + )), + } } } diff --git a/crates/rendering/src/lib.rs b/crates/rendering/src/lib.rs index b781874bf5..4da4bc794f 100644 --- a/crates/rendering/src/lib.rs +++ b/crates/rendering/src/lib.rs @@ -40,7 +40,7 @@ pub mod yuv_converter; mod zoom; pub use coord::*; -pub use decoder::{DecodedFrame, PixelFormat}; +pub use decoder::{DecodedFrame, DecoderStatus, DecoderType, PixelFormat}; pub use frame_pipeline::RenderedFrame; pub use project_recordings::{ProjectRecordingsMeta, SegmentRecordings, Video}; diff --git a/crates/scap-direct3d/Cargo.toml b/crates/scap-direct3d/Cargo.toml index d19e4e8237..928ce647c1 100644 --- a/crates/scap-direct3d/Cargo.toml +++ b/crates/scap-direct3d/Cargo.toml @@ -29,6 +29,8 @@ windows = { workspace = true, features = [ "Win32_System_Variant", "Storage_Search", "Storage_Streams", + "Win32_System_SystemInformation", + "Win32_System_LibraryLoader", ] } [dev-dependencies] @@ -41,4 +43,5 @@ workspace = true [dependencies] thiserror.workspace = true +tracing.workspace = true workspace-hack = { version = "0.1", path = "../workspace-hack" } diff --git a/crates/scap-direct3d/examples/cli.rs b/crates/scap-direct3d/examples/cli.rs index cc8020267b..faaa65a547 100644 --- a/crates/scap-direct3d/examples/cli.rs +++ b/crates/scap-direct3d/examples/cli.rs @@ -5,15 +5,12 @@ fn main() { #[cfg(windows)] mod windows { - use scap_direct3d::{Capturer, PixelFormat, Settings}; - use scap_ffmpeg::*; use scap_targets::*; use std::time::Duration; - use windows::Win32::Graphics::Direct3D11::D3D11_BOX; pub fn main() { let display = Display::primary(); - let display = display.raw_handle(); + let _display = display.raw_handle(); // let mut capturer = Capturer::new( // display.try_as_capture_item().unwrap(), diff --git a/crates/scap-direct3d/src/lib.rs b/crates/scap-direct3d/src/lib.rs index ac704e934f..c631f149af 100644 --- a/crates/scap-direct3d/src/lib.rs +++ b/crates/scap-direct3d/src/lib.rs @@ -1,11 +1,13 @@ -// a whole bunch of credit to https://github.com/NiiightmareXD/windows-capture - #![cfg(windows)] +mod windows_version; + +pub use windows_version::WindowsVersion; + use std::{ sync::{ - Arc, - atomic::{AtomicBool, Ordering}, + Arc, Mutex, + atomic::{AtomicBool, AtomicUsize, Ordering}, mpsc::RecvError, }, time::Duration, @@ -22,10 +24,10 @@ use windows::{ Win32::{ Foundation::HMODULE, Graphics::{ - Direct3D::D3D_DRIVER_TYPE_HARDWARE, + Direct3D::{D3D_DRIVER_TYPE, D3D_DRIVER_TYPE_HARDWARE, D3D_DRIVER_TYPE_WARP}, Direct3D11::{ D3D11_BIND_RENDER_TARGET, D3D11_BIND_SHADER_RESOURCE, D3D11_BOX, - D3D11_CPU_ACCESS_READ, D3D11_CPU_ACCESS_WRITE, D3D11_MAP_READ_WRITE, + D3D11_CPU_ACCESS_READ, D3D11_CREATE_DEVICE_FLAG, D3D11_MAP_READ, D3D11_MAPPED_SUBRESOURCE, D3D11_SDK_VERSION, D3D11_TEXTURE2D_DESC, D3D11_USAGE_DEFAULT, D3D11_USAGE_STAGING, D3D11CreateDevice, ID3D11Device, ID3D11DeviceContext, ID3D11Texture2D, @@ -35,7 +37,7 @@ use windows::{ DXGI_FORMAT, DXGI_FORMAT_B8G8R8A8_UNORM, DXGI_FORMAT_R8G8B8A8_UNORM, DXGI_SAMPLE_DESC, }, - IDXGIDevice, + DXGI_ERROR_UNSUPPORTED, IDXGIDevice, }, }, System::WinRT::Direct3D11::{ @@ -69,6 +71,89 @@ impl PixelFormat { } } +const STAGING_POOL_SIZE: usize = 3; + +struct PooledStagingTexture { + texture: ID3D11Texture2D, + width: u32, + height: u32, +} + +pub struct StagingTexturePool { + textures: Mutex>, + d3d_device: ID3D11Device, + pixel_format: PixelFormat, + next_index: AtomicUsize, +} + +impl StagingTexturePool { + fn new(d3d_device: ID3D11Device, pixel_format: PixelFormat) -> Self { + Self { + textures: Mutex::new(Vec::with_capacity(STAGING_POOL_SIZE)), + d3d_device, + pixel_format, + next_index: AtomicUsize::new(0), + } + } + + fn get_or_create_texture( + &self, + width: u32, + height: u32, + ) -> windows::core::Result { + let mut textures = self.textures.lock().unwrap(); + + let index = self.next_index.fetch_add(1, Ordering::Relaxed) % STAGING_POOL_SIZE; + + if let Some(pooled) = textures.get(index) + && pooled.width == width + && pooled.height == height + { + return Ok(pooled.texture.clone()); + } + + let texture_desc = D3D11_TEXTURE2D_DESC { + Width: width, + Height: height, + MipLevels: 1, + ArraySize: 1, + Format: self.pixel_format.as_dxgi(), + SampleDesc: DXGI_SAMPLE_DESC { + Count: 1, + Quality: 0, + }, + Usage: D3D11_USAGE_STAGING, + BindFlags: 0, + CPUAccessFlags: D3D11_CPU_ACCESS_READ.0 as u32, + MiscFlags: 0, + }; + + let mut texture = None; + unsafe { + self.d3d_device + .CreateTexture2D(&texture_desc, None, Some(&mut texture))?; + }; + + let texture = texture.unwrap(); + + if index < textures.len() { + textures[index] = PooledStagingTexture { + texture: texture.clone(), + width, + height, + }; + } else { + textures.push(PooledStagingTexture { + texture: texture.clone(), + width, + height, + }); + } + + Ok(texture) + } +} + pub fn is_supported() -> windows::core::Result { Ok(ApiInformation::IsApiContractPresentByMajor( &HSTRING::from("Windows.Foundation.UniversalApiContract"), @@ -76,6 +161,43 @@ pub fn is_supported() -> windows::core::Result { )? && GraphicsCaptureSession::IsSupported()?) } +fn create_d3d_device_with_type( + driver_type: D3D_DRIVER_TYPE, + flags: D3D11_CREATE_DEVICE_FLAG, + device: *mut Option, +) -> windows::core::Result<()> { + unsafe { + D3D11CreateDevice( + None, + driver_type, + HMODULE::default(), + flags, + None, + D3D11_SDK_VERSION, + Some(device), + None, + None, + ) + } +} + +fn create_d3d_device_with_warp_fallback() -> windows::core::Result<(ID3D11Device, bool)> { + let mut device = None; + let flags = D3D11_CREATE_DEVICE_FLAG::default(); + + let result = create_d3d_device_with_type(D3D_DRIVER_TYPE_HARDWARE, flags, &mut device); + + match result { + Ok(()) => Ok((device.unwrap(), false)), + Err(e) if e.code() == DXGI_ERROR_UNSUPPORTED => { + tracing::info!("Hardware D3D11 device unavailable, attempting WARP fallback"); + create_d3d_device_with_type(D3D_DRIVER_TYPE_WARP, flags, &mut device)?; + Ok((device.unwrap(), true)) + } + Err(e) => Err(e), + } +} + #[derive(Clone, Default, Debug)] pub struct Settings { pub is_border_required: Option, @@ -83,6 +205,7 @@ pub struct Settings { pub min_update_interval: Option, pub pixel_format: PixelFormat, pub crop: Option, + pub fps: Option, } impl Settings { @@ -110,6 +233,12 @@ impl Settings { #[derive(Clone, Debug, thiserror::Error)] pub enum NewCapturerError { + #[error("Screen capture requires Windows 10 version 1903 (build 18362) or later")] + WindowsVersionTooOld, + #[error( + "Windows Graphics Capture API is disabled or unavailable. This may be due to group policy or missing system components." + )] + GraphicsCaptureDisabled, #[error("NotSupported")] NotSupported, #[error("BorderNotSupported")] @@ -150,6 +279,7 @@ pub struct Capturer { frame_pool: Direct3D11CaptureFramePool, frame_arrived_token: i64, stop_flag: Arc, + is_using_warp: bool, } impl Capturer { @@ -158,10 +288,37 @@ impl Capturer { settings: Settings, mut callback: impl FnMut(Frame) -> windows::core::Result<()> + Send + 'static, mut closed_callback: impl FnMut() -> windows::core::Result<()> + Send + 'static, - mut d3d_device: Option, + d3d_device: Option, ) -> Result { - if !is_supported()? { - return Err(NewCapturerError::NotSupported); + if let Some(version) = WindowsVersion::detect() { + tracing::debug!( + version = %version.display_name(), + meets_requirements = version.meets_minimum_requirements(), + "Initializing screen capture" + ); + + if !version.meets_minimum_requirements() { + tracing::error!( + version = %version.display_name(), + required = "Windows 10 version 1903 (build 18362)", + "Windows version does not meet minimum requirements" + ); + return Err(NewCapturerError::WindowsVersionTooOld); + } + } + + let api_present = ApiInformation::IsApiContractPresentByMajor( + &HSTRING::from("Windows.Foundation.UniversalApiContract"), + 8, + ) + .unwrap_or(false); + + if !api_present { + return Err(NewCapturerError::WindowsVersionTooOld); + } + + if !GraphicsCaptureSession::IsSupported().unwrap_or(false) { + return Err(NewCapturerError::GraphicsCaptureDisabled); } if settings.is_border_required.is_some() && !Settings::can_is_border_required()? { @@ -178,29 +335,23 @@ impl Capturer { return Err(NewCapturerError::UpdateIntervalNotSupported); } - if d3d_device.is_none() { - unsafe { - D3D11CreateDevice( - None, - D3D_DRIVER_TYPE_HARDWARE, - HMODULE::default(), - Default::default(), - None, - D3D11_SDK_VERSION, - Some(&mut d3d_device), - None, - None, - ) - } - .map_err(NewCapturerError::CreateDevice)?; - } + let (d3d_device, is_using_warp) = if let Some(device) = d3d_device { + (device, false) + } else { + create_d3d_device_with_warp_fallback().map_err(NewCapturerError::CreateDevice)? + }; - let (d3d_device, d3d_context) = d3d_device + let (d3d_device, d3d_context) = Some(d3d_device) .map(|d| unsafe { d.GetImmediateContext() }.map(|v| (d, v))) .transpose() .map_err(NewCapturerError::Context)? .unwrap(); + let staging_pool = Arc::new(StagingTexturePool::new( + d3d_device.clone(), + settings.pixel_format, + )); + let item = item.clone(); let settings = settings.clone(); let stop_flag = Arc::new(AtomicBool::new(false)); @@ -212,10 +363,15 @@ impl Capturer { })() .map_err(NewCapturerError::Direct3DDevice)?; + let frame_pool_size = settings + .fps + .map(|fps| ((fps as f32 / 30.0 * 2.0).ceil() as i32).clamp(2, 4)) + .unwrap_or(2); + let frame_pool = Direct3D11CaptureFramePool::CreateFreeThreaded( &direct3d_device, settings.pixel_format.as_directx(), - 1, + frame_pool_size, item.Size().map_err(NewCapturerError::ItemSize)?, ) .map_err(NewCapturerError::FramePool)?; @@ -273,6 +429,7 @@ impl Capturer { let d3d_context = d3d_context.clone(); let d3d_device = d3d_device.clone(); let stop_flag = stop_flag.clone(); + let staging_pool = staging_pool.clone(); move |frame_pool, _| { if stop_flag.load(Ordering::Relaxed) { @@ -312,6 +469,7 @@ impl Capturer { texture: cropped_texture, d3d_context: d3d_context.clone(), d3d_device: d3d_device.clone(), + staging_pool: staging_pool.clone(), } } else { Frame { @@ -322,6 +480,7 @@ impl Capturer { texture, d3d_context: d3d_context.clone(), d3d_device: d3d_device.clone(), + staging_pool: staging_pool.clone(), } }; @@ -338,6 +497,12 @@ impl Capturer { ) .map_err(NewCapturerError::RegisterClosed)?; + if is_using_warp { + tracing::warn!( + "Hardware GPU unavailable, using WARP software rasterizer for screen capture" + ); + } + Ok(Capturer { settings, d3d_device, @@ -346,9 +511,14 @@ impl Capturer { frame_pool, frame_arrived_token, stop_flag, + is_using_warp, }) } + pub fn is_using_software_rendering(&self) -> bool { + self.is_using_warp + } + pub fn settings(&self) -> &Settings { &self.settings } @@ -407,6 +577,7 @@ pub struct Frame { texture: ID3D11Texture2D, d3d_device: ID3D11Device, d3d_context: ID3D11DeviceContext, + staging_pool: Arc, } impl std::fmt::Debug for Frame { @@ -447,49 +618,29 @@ impl Frame { &self.d3d_context } - pub fn as_buffer<'a>(&'a self) -> windows::core::Result> { - let texture_desc = D3D11_TEXTURE2D_DESC { - Width: self.width, - Height: self.height, - MipLevels: 1, - ArraySize: 1, - Format: self.pixel_format.as_dxgi(), - SampleDesc: DXGI_SAMPLE_DESC { - Count: 1, - Quality: 0, - }, - Usage: D3D11_USAGE_STAGING, - BindFlags: 0, - CPUAccessFlags: D3D11_CPU_ACCESS_READ.0 as u32 | D3D11_CPU_ACCESS_WRITE.0 as u32, - MiscFlags: 0, - }; + pub fn as_buffer(&self) -> windows::core::Result> { + let staging_texture = self + .staging_pool + .get_or_create_texture(self.width, self.height)?; - let mut texture = None; unsafe { - self.d3d_device - .CreateTexture2D(&texture_desc, None, Some(&mut texture))?; - }; - - let texture = texture.unwrap(); - - // Copies GPU only texture to CPU-mappable texture - unsafe { - self.d3d_context.CopyResource(&texture, &self.texture); + self.d3d_context + .CopyResource(&staging_texture, &self.texture); }; let mut mapped_resource = D3D11_MAPPED_SUBRESOURCE::default(); unsafe { self.d3d_context.Map( - &texture, + &staging_texture, 0, - D3D11_MAP_READ_WRITE, + D3D11_MAP_READ, 0, Some(&mut mapped_resource), )?; }; let data = unsafe { - std::slice::from_raw_parts_mut( + std::slice::from_raw_parts( mapped_resource.pData.cast(), (self.height * mapped_resource.RowPitch) as usize, ) @@ -501,21 +652,31 @@ impl Frame { height: self.height, stride: mapped_resource.RowPitch, pixel_format: self.pixel_format, - resource: mapped_resource, + staging_texture, + d3d_context: self.d3d_context.clone(), }) } } pub struct FrameBuffer<'a> { - data: &'a mut [u8], + data: &'a [u8], width: u32, height: u32, stride: u32, - resource: D3D11_MAPPED_SUBRESOURCE, pixel_format: PixelFormat, + staging_texture: ID3D11Texture2D, + d3d_context: ID3D11DeviceContext, +} + +impl Drop for FrameBuffer<'_> { + fn drop(&mut self) { + unsafe { + self.d3d_context.Unmap(&self.staging_texture, 0); + } + } } -impl<'a> FrameBuffer<'a> { +impl FrameBuffer<'_> { pub fn width(&self) -> u32 { self.width } @@ -532,10 +693,6 @@ impl<'a> FrameBuffer<'a> { self.data } - pub fn inner(&self) -> &D3D11_MAPPED_SUBRESOURCE { - &self.resource - } - pub fn pixel_format(&self) -> PixelFormat { self.pixel_format } diff --git a/crates/scap-direct3d/src/windows_version.rs b/crates/scap-direct3d/src/windows_version.rs new file mode 100644 index 0000000000..100aaa3588 --- /dev/null +++ b/crates/scap-direct3d/src/windows_version.rs @@ -0,0 +1,159 @@ +#![cfg(windows)] + +use std::sync::OnceLock; +use windows::Win32::System::SystemInformation::OSVERSIONINFOEXW; + +static DETECTED_VERSION: OnceLock> = OnceLock::new(); + +type RtlGetVersionFn = unsafe extern "system" fn(*mut OSVERSIONINFOEXW) -> i32; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct WindowsVersion { + pub major: u32, + pub minor: u32, + pub build: u32, +} + +impl WindowsVersion { + pub fn detect() -> Option { + *DETECTED_VERSION.get_or_init(detect_version_internal) + } + + pub fn meets_minimum_requirements(&self) -> bool { + self.major > 10 || (self.major == 10 && self.build >= 18362) + } + + pub fn supports_border_control(&self) -> bool { + self.build >= 22000 + } + + pub fn is_windows_11(&self) -> bool { + self.build >= 22000 + } + + pub fn display_name(&self) -> String { + if self.build >= 22000 { + format!("Windows 11 (Build {})", self.build) + } else if self.major == 10 { + format!("Windows 10 (Build {})", self.build) + } else { + format!( + "Windows {}.{} (Build {})", + self.major, self.minor, self.build + ) + } + } +} + +fn detect_version_internal() -> Option { + unsafe { + let ntdll = + windows::Win32::System::LibraryLoader::GetModuleHandleW(windows::core::w!("ntdll.dll")) + .ok()?; + + let rtl_get_version: RtlGetVersionFn = + std::mem::transmute(windows::Win32::System::LibraryLoader::GetProcAddress( + ntdll, + windows::core::s!("RtlGetVersion"), + )?); + + let mut info = OSVERSIONINFOEXW { + dwOSVersionInfoSize: std::mem::size_of::() as u32, + ..Default::default() + }; + + if rtl_get_version(&mut info) == 0 { + let version = WindowsVersion { + major: info.dwMajorVersion, + minor: info.dwMinorVersion, + build: info.dwBuildNumber, + }; + + tracing::debug!( + major = version.major, + minor = version.minor, + build = version.build, + display_name = %version.display_name(), + "Detected Windows version via RtlGetVersion" + ); + + return Some(version); + } + + tracing::warn!("RtlGetVersion failed"); + None + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_detect_returns_some() { + let version = WindowsVersion::detect(); + assert!(version.is_some(), "Should detect Windows version"); + } + + #[test] + fn test_version_requirements() { + let old_version = WindowsVersion { + major: 10, + minor: 0, + build: 17000, + }; + assert!(!old_version.meets_minimum_requirements()); + + let min_version = WindowsVersion { + major: 10, + minor: 0, + build: 18362, + }; + assert!(min_version.meets_minimum_requirements()); + + let new_version = WindowsVersion { + major: 10, + minor: 0, + build: 19041, + }; + assert!(new_version.meets_minimum_requirements()); + } + + #[test] + fn test_windows_11_detection() { + let win10 = WindowsVersion { + major: 10, + minor: 0, + build: 19045, + }; + assert!(!win10.is_windows_11()); + assert!(!win10.supports_border_control()); + + let win11 = WindowsVersion { + major: 10, + minor: 0, + build: 22000, + }; + assert!(win11.is_windows_11()); + assert!(win11.supports_border_control()); + } + + #[test] + fn test_display_name() { + let win10 = WindowsVersion { + major: 10, + minor: 0, + build: 19045, + }; + assert!(win10.display_name().contains("Windows 10")); + assert!(win10.display_name().contains("19045")); + + let win11 = WindowsVersion { + major: 10, + minor: 0, + build: 22631, + }; + assert!(win11.display_name().contains("Windows 11")); + assert!(win11.display_name().contains("22631")); + } +} diff --git a/crates/scap-ffmpeg/src/direct3d.rs b/crates/scap-ffmpeg/src/direct3d.rs index 0aec041f02..fcfdd34fa6 100644 --- a/crates/scap-ffmpeg/src/direct3d.rs +++ b/crates/scap-ffmpeg/src/direct3d.rs @@ -3,6 +3,49 @@ use scap_direct3d::PixelFormat; pub type AsFFmpegError = windows::core::Error; +#[inline] +fn copy_frame_data( + src_bytes: &[u8], + src_stride: usize, + dest_bytes: &mut [u8], + dest_stride: usize, + row_length: usize, + height: usize, +) { + debug_assert!(height > 0, "height must be positive"); + debug_assert!( + src_bytes.len() + >= (height - 1) + .saturating_mul(src_stride) + .saturating_add(row_length), + "source buffer too small" + ); + debug_assert!( + dest_bytes.len() + >= (height - 1) + .saturating_mul(dest_stride) + .saturating_add(row_length), + "destination buffer too small" + ); + + if src_stride == row_length && dest_stride == row_length { + let total_bytes = row_length.saturating_mul(height); + unsafe { + std::ptr::copy_nonoverlapping(src_bytes.as_ptr(), dest_bytes.as_mut_ptr(), total_bytes); + } + } else { + for i in 0..height { + unsafe { + std::ptr::copy_nonoverlapping( + src_bytes.as_ptr().add(i * src_stride), + dest_bytes.as_mut_ptr().add(i * dest_stride), + row_length, + ); + } + } + } +} + impl super::AsFFmpeg for scap_direct3d::Frame { fn as_ffmpeg(&self) -> Result { let buffer = self.as_buffer()?; @@ -12,6 +55,7 @@ impl super::AsFFmpeg for scap_direct3d::Frame { let src_bytes = buffer.data(); let src_stride = buffer.stride() as usize; + let row_length = width * 4; match self.pixel_format() { PixelFormat::R8G8B8A8Unorm => { @@ -24,14 +68,14 @@ impl super::AsFFmpeg for scap_direct3d::Frame { let dest_stride = ff_frame.stride(0); let dest_bytes = ff_frame.data_mut(0); - let row_length = width * 4; - - for i in 0..height { - let src_row = &src_bytes[i * src_stride..i * src_stride + row_length]; - let dest_row = &mut dest_bytes[i * dest_stride..i * dest_stride + row_length]; - - dest_row.copy_from_slice(src_row); - } + copy_frame_data( + src_bytes, + src_stride, + dest_bytes, + dest_stride, + row_length, + height, + ); Ok(ff_frame) } @@ -45,14 +89,14 @@ impl super::AsFFmpeg for scap_direct3d::Frame { let dest_stride = ff_frame.stride(0); let dest_bytes = ff_frame.data_mut(0); - let row_length = width * 4; - - for i in 0..height { - let src_row = &src_bytes[i * src_stride..i * src_stride + row_length]; - let dest_row = &mut dest_bytes[i * dest_stride..i * dest_stride + row_length]; - - dest_row.copy_from_slice(src_row); - } + copy_frame_data( + src_bytes, + src_stride, + dest_bytes, + dest_stride, + row_length, + height, + ); Ok(ff_frame) } diff --git a/crates/video-decode/src/ffmpeg.rs b/crates/video-decode/src/ffmpeg.rs index 354c71e935..7a7a69f296 100644 --- a/crates/video-decode/src/ffmpeg.rs +++ b/crates/video-decode/src/ffmpeg.rs @@ -281,6 +281,10 @@ impl FFmpegDecoder { pub fn start_time(&self) -> i64 { self.start_time } + + pub fn is_hardware_accelerated(&self) -> bool { + self.hw_device.is_some() + } } unsafe impl Send for FFmpegDecoder {}