Add support for more pixel formats and GPU converters

.claude/settings.local.json

@@ -41,7 +41,8 @@
 			"Bash(rustfmt:*)",
 			"Bash(cargo tree:*)",
 			"WebFetch(domain:github.com)",
-			"WebFetch(domain:docs.rs)"
+			"WebFetch(domain:docs.rs)",
+			"WebFetch(domain:gix.github.io)"
 		],
 		"deny": [],
 		"ask": []

crates/camera-ffmpeg/Cargo.toml

@@ -7,6 +7,7 @@ license = "MIT"
 [dependencies]
 ffmpeg = { workspace = true }
 thiserror.workspace = true
+tracing.workspace = true
 cap-camera = { path = "../camera" }
 workspace-hack = { version = "0.1", path = "../workspace-hack" }
 

crates/camera-ffmpeg/src/macos.rs

@@ -1,6 +1,8 @@
 use cap_camera::CapturedFrame;
 use cap_camera_avfoundation::ImageBufExt;
 use cidre::*;
+use ffmpeg::{format::Pixel, software::scaling};
+use std::sync::atomic::{AtomicBool, Ordering};
 
 use crate::CapturedFrameExt;
 
@@ -14,10 +16,41 @@ pub enum AsFFmpegError {
         expected: usize,
         found: usize,
     },
+    #[error("Swscale fallback failed for format '{format}': {reason}")]
+    SwscaleFallbackFailed { format: String, reason: String },
     #[error("{0}")]
     Native(#[from] cidre::os::Error),
 }
 
+struct FourccInfo {
+    pixel: Pixel,
+    bytes_per_pixel: usize,
+}
+
+fn fourcc_to_pixel_format(fourcc: &str) -> Option<FourccInfo> {
+    match fourcc {
+        "ABGR" => Some(FourccInfo {
+            pixel: Pixel::ABGR,
+            bytes_per_pixel: 4,
+        }),
+        "b64a" => Some(FourccInfo {
+            pixel: Pixel::RGBA64BE,
+            bytes_per_pixel: 8,
+        }),
+        "b48r" => Some(FourccInfo {
+            pixel: Pixel::RGB48BE,
+            bytes_per_pixel: 6,
+        }),
+        "L016" => Some(FourccInfo {
+            pixel: Pixel::GRAY16LE,
+            bytes_per_pixel: 2,
+        }),
+        _ => None,
+    }
+}
+
+static FALLBACK_WARNING_LOGGED: AtomicBool = AtomicBool::new(false);
+
 impl CapturedFrameExt for CapturedFrame {
     fn as_ffmpeg(&self) -> Result<ffmpeg::frame::Video, AsFFmpegError> {
         let native = self.native().clone();
@@ -162,6 +195,29 @@ impl CapturedFrameExt for CapturedFrame {
 
                 ff_frame
             }
+            "RGBA" => {
+                let mut ff_frame = ffmpeg::frame::Video::new(
+                    ffmpeg::format::Pixel::RGBA,
+                    width as u32,
+                    height as u32,
+                );
+
+                let src_stride = native.image_buf().plane_bytes_per_row(0);
+                let dest_stride = ff_frame.stride(0);
+
+                let src_bytes = bytes_lock.plane_data(0);
+                let dest_bytes = &mut ff_frame.data_mut(0);
+
+                for y in 0..height {
+                    let row_width = width * 4;
+                    let src_row = &src_bytes[y * src_stride..y * src_stride + row_width];
+                    let dest_row = &mut dest_bytes[y * dest_stride..y * dest_stride + row_width];
+
+                    dest_row.copy_from_slice(src_row);
+                }
+
+                ff_frame
+            }
             "24BG" => {
                 let mut ff_frame = ffmpeg::frame::Video::new(
                     ffmpeg::format::Pixel::BGR24,
@@ -185,6 +241,29 @@ impl CapturedFrameExt for CapturedFrame {
 
                 ff_frame
             }
+            "24RG" => {
+                let mut ff_frame = ffmpeg::frame::Video::new(
+                    ffmpeg::format::Pixel::RGB24,
+                    width as u32,
+                    height as u32,
+                );
+
+                let src_stride = native.image_buf().plane_bytes_per_row(0);
+                let dest_stride = ff_frame.stride(0);
+
+                let src_bytes = bytes_lock.plane_data(0);
+                let dest_bytes = &mut ff_frame.data_mut(0);
+
+                for y in 0..height {
+                    let row_width = width * 3;
+                    let src_row = &src_bytes[y * src_stride..y * src_stride + row_width];
+                    let dest_row = &mut dest_bytes[y * dest_stride..y * dest_stride + row_width];
+
+                    dest_row.copy_from_slice(src_row);
+                }
+
+                ff_frame
+            }
             "y420" => {
                 let plane_count = native.image_buf().plane_count();
                 if plane_count < 3 {
@@ -220,8 +299,82 @@ impl CapturedFrameExt for CapturedFrame {
 
                 ff_frame
             }
+            "L008" | "GRAY" => {
+                let mut ff_frame = ffmpeg::frame::Video::new(
+                    ffmpeg::format::Pixel::GRAY8,
+                    width as u32,
+                    height as u32,
+                );
+
+                let src_stride = native.image_buf().plane_bytes_per_row(0);
+                let dest_stride = ff_frame.stride(0);
+
+                let src_bytes = bytes_lock.plane_data(0);
+                let dest_bytes = &mut ff_frame.data_mut(0);
+
+                for y in 0..height {
+                    let row_width = width;
+                    let src_row = &src_bytes[y * src_stride..y * src_stride + row_width];
+                    let dest_row = &mut dest_bytes[y * dest_stride..y * dest_stride + row_width];
+
+                    dest_row.copy_from_slice(src_row);
+                }
+
+                ff_frame
+            }
             format => {
-                return Err(AsFFmpegError::UnsupportedSubType(format.to_string()));
+                if let Some(info) = fourcc_to_pixel_format(format) {
+                    if !FALLBACK_WARNING_LOGGED.swap(true, Ordering::Relaxed) {
+                        tracing::warn!(
+                            "Using swscale fallback for camera format '{}' - this may impact performance",
+                            format
+                        );
+                    }
+
+                    let mut src_frame =
+                        ffmpeg::frame::Video::new(info.pixel, width as u32, height as u32);
+
+                    let src_stride = native.image_buf().plane_bytes_per_row(0);
+                    let dest_stride = src_frame.stride(0);
+                    let src_bytes = bytes_lock.plane_data(0);
+                    let dest_bytes = &mut src_frame.data_mut(0);
+
+                    let row_width = width * info.bytes_per_pixel;
+                    for y in 0..height {
+                        let src_row = &src_bytes[y * src_stride..y * src_stride + row_width];
+                        let dest_row =
+                            &mut dest_bytes[y * dest_stride..y * dest_stride + row_width];
+                        dest_row.copy_from_slice(src_row);
+                    }
+
+                    let mut scaler = scaling::Context::get(
+                        info.pixel,
+                        width as u32,
+                        height as u32,
+                        Pixel::RGBA,
+                        width as u32,
+                        height as u32,
+                        scaling::flag::Flags::FAST_BILINEAR,
+                    )
+                    .map_err(|e| AsFFmpegError::SwscaleFallbackFailed {
+                        format: format.to_string(),
+                        reason: format!("Failed to create scaler: {e}"),
+                    })?;
+
+                    let mut output_frame =
+                        ffmpeg::frame::Video::new(Pixel::RGBA, width as u32, height as u32);
+
+                    scaler.run(&src_frame, &mut output_frame).map_err(|e| {
+                        AsFFmpegError::SwscaleFallbackFailed {
+                            format: format.to_string(),
+                            reason: format!("Conversion failed: {e}"),
+                        }
+                    })?;
+
+                    output_frame
+                } else {
+                    return Err(AsFFmpegError::UnsupportedSubType(format.to_string()));
+                }
             }
         };
 

crates/camera-ffmpeg/src/windows.rs

@@ -12,6 +12,8 @@ pub enum AsFFmpegError {
     Empty,
     #[error("MJPEG decode error: {0}")]
     MjpegDecodeError(String),
+    #[error("H264 decode error: {0}")]
+    H264DecodeError(String),
 }
 
 fn decode_mjpeg(bytes: &[u8]) -> Result<FFVideo, AsFFmpegError> {
@@ -38,6 +40,30 @@ fn decode_mjpeg(bytes: &[u8]) -> Result<FFVideo, AsFFmpegError> {
     Ok(decoded_frame)
 }
 
+fn decode_h264(bytes: &[u8]) -> Result<FFVideo, AsFFmpegError> {
+    let codec = ffmpeg::codec::decoder::find(ffmpeg::codec::Id::H264)
+        .ok_or_else(|| AsFFmpegError::H264DecodeError("H264 codec not found".to_string()))?;
+
+    let decoder_context = ffmpeg::codec::context::Context::new_with_codec(codec);
+
+    let mut decoder = decoder_context
+        .decoder()
+        .video()
+        .map_err(|e| AsFFmpegError::H264DecodeError(format!("Failed to create decoder: {e}")))?;
+
+    let packet = Packet::copy(bytes);
+    decoder
+        .send_packet(&packet)
+        .map_err(|e| AsFFmpegError::H264DecodeError(format!("Failed to send packet: {e}")))?;
+
+    let mut decoded_frame = FFVideo::empty();
+    decoder
+        .receive_frame(&mut decoded_frame)
+        .map_err(|e| AsFFmpegError::H264DecodeError(format!("Failed to receive frame: {e}")))?;
+
+    Ok(decoded_frame)
+}
+
 impl CapturedFrameExt for CapturedFrame {
     fn as_ffmpeg(&self) -> Result<ffmpeg::frame::Video, AsFFmpegError> {
         let native = self.native();
@@ -232,6 +258,98 @@ impl CapturedFrameExt for CapturedFrame {
 
                 ff_frame
             }
+            PixelFormat::GRAY8 => {
+                let mut ff_frame = FFVideo::new(Pixel::GRAY8, width as u32, height as u32);
+
+                let stride = ff_frame.stride(0);
+
+                for y in 0..height {
+                    let row_width = width;
+                    let src_row = &bytes[y * row_width..];
+                    let dest_row = &mut ff_frame.data_mut(0)[y * stride..];
+                    dest_row[0..row_width].copy_from_slice(&src_row[0..row_width]);
+                }
+
+                ff_frame
+            }
+            PixelFormat::GRAY16 => {
+                let mut ff_frame = FFVideo::new(Pixel::GRAY16LE, width as u32, height as u32);
+
+                let stride = ff_frame.stride(0);
+                let src_stride = width * 2;
+
+                for y in 0..height {
+                    let src_row = &bytes[y * src_stride..];
+                    let dest_row = &mut ff_frame.data_mut(0)[y * stride..];
+                    dest_row[0..src_stride].copy_from_slice(&src_row[0..src_stride]);
+                }
+
+                ff_frame
+            }
+            PixelFormat::NV21 => {
+                let mut ff_frame = FFVideo::new(Pixel::NV12, width as u32, height as u32);
+
+                let stride = ff_frame.stride(0);
+                for y in 0..height {
+                    let src_row = &bytes[y * width..];
+                    let dest_row = &mut ff_frame.data_mut(0)[y * stride..];
+                    dest_row[0..width].copy_from_slice(&src_row[0..width]);
+                }
+
+                let stride = ff_frame.stride(1);
+                let src_uv = &bytes[width * height..];
+
+                for y in 0..height / 2 {
+                    let row_width = width;
+                    let src_row = &src_uv[y * row_width..];
+                    let dest_row = &mut ff_frame.data_mut(1)[y * stride..];
+                    for x in 0..width / 2 {
+                        dest_row[x * 2] = src_row[x * 2 + 1];
+                        dest_row[x * 2 + 1] = src_row[x * 2];
+                    }
+                }
+
+                ff_frame
+            }
+            PixelFormat::RGB565 => {
+                let mut ff_frame = FFVideo::new(Pixel::RGB565LE, width as u32, height as u32);
+
+                let stride = ff_frame.stride(0);
+                let src_stride = width * 2;
+
+                for y in 0..height {
+                    let src_row = &bytes[(height - y - 1) * src_stride..];
+                    let dest_row = &mut ff_frame.data_mut(0)[y * stride..];
+                    dest_row[0..src_stride].copy_from_slice(&src_row[0..src_stride]);
+                }
+
+                ff_frame
+            }
+            PixelFormat::P010 => {
+                let mut ff_frame = FFVideo::new(Pixel::P010LE, width as u32, height as u32);
+
+                let stride = ff_frame.stride(0);
+                let src_stride = width * 2;
+
+                for y in 0..height {
+                    let src_row = &bytes[y * src_stride..];
+                    let dest_row = &mut ff_frame.data_mut(0)[y * stride..];
+                    dest_row[0..src_stride].copy_from_slice(&src_row[0..src_stride]);
+                }
+
+                let stride = ff_frame.stride(1);
+                let uv_offset = width * height * 2;
+                let src_stride = width * 2;
+
+                for y in 0..height / 2 {
+                    let src_row = &bytes[uv_offset + y * src_stride..];
+                    let dest_row = &mut ff_frame.data_mut(1)[y * stride..];
+                    dest_row[0..src_stride].copy_from_slice(&src_row[0..src_stride]);
+                }
+
+                ff_frame
+            }
+            PixelFormat::H264 => decode_h264(&bytes)?,
         })
     }
 }