diff --git a/Cargo.lock b/Cargo.lock index 99d9d02..b25d8c4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1497,7 +1497,7 @@ dependencies = [ [[package]] name = "cosmic-client-toolkit" version = "0.1.0" -source = "git+https://github.com/pop-os/cosmic-protocols//?branch=main#6254f50abc6dbfccadc6939f80e20081ab5f9d51" +source = "git+https://github.com/pop-os/cosmic-protocols//?branch=main#af1997b1827ad64aab46fa31c0b77fb20d7a537a" dependencies = [ "bitflags 2.9.4", "cosmic-protocols", @@ -1618,7 +1618,7 @@ dependencies = [ [[package]] name = "cosmic-protocols" version = "0.1.0" -source = "git+https://github.com/pop-os/cosmic-protocols//?branch=main#6254f50abc6dbfccadc6939f80e20081ab5f9d51" +source = "git+https://github.com/pop-os/cosmic-protocols//?branch=main#af1997b1827ad64aab46fa31c0b77fb20d7a537a" dependencies = [ "bitflags 2.9.4", "wayland-backend", diff --git a/src/screencast.rs b/src/screencast.rs index 8646d98..7b8a04c 100644 --- a/src/screencast.rs +++ b/src/screencast.rs @@ -352,7 +352,7 @@ impl ScreenCast { #[zbus(property)] async fn available_cursor_modes(&self) -> u32 { // TODO: Support metadata? - CURSOR_MODE_HIDDEN | CURSOR_MODE_EMBEDDED + CURSOR_MODE_HIDDEN | CURSOR_MODE_EMBEDDED | CURSOR_MODE_METADATA } #[zbus(property, name = "version")] diff --git a/src/screencast_thread.rs b/src/screencast_thread.rs index 7ebf6d0..29cf384 100644 --- a/src/screencast_thread.rs +++ b/src/screencast_thread.rs @@ -5,7 +5,7 @@ // Dmabuf modifier negotiation is described in https://docs.pipewire.org/page_dma_buf.html use cosmic_client_toolkit::screencopy::{FailureReason, Formats, Rect}; -use futures::executor::block_on; +use futures::{executor::block_on, stream::StreamExt}; use pipewire::{ spa::{ self, @@ -15,7 +15,14 @@ use pipewire::{ stream::{StreamRef, StreamState}, sys::pw_buffer, }; -use std::{collections::HashMap, ffi::c_void, io, iter, os::fd::IntoRawFd, slice}; +use std::{ + collections::HashMap, + ffi::c_void, + io, iter, + os::fd::IntoRawFd, + slice, + task::{Context, Poll, Waker}, +}; use tokio::sync::oneshot; use wayland_client::{ WEnum, @@ -24,7 +31,7 @@ use wayland_client::{ use crate::{ buffer, - wayland::{CaptureSource, DmabufHelper, Session, WaylandHelper}, + wayland::{CaptureSource, CursorStream, DmabufHelper, Session, WaylandHelper}, }; static FORMAT_MAP: &[(gbm::Format, Id)] = &[ @@ -56,6 +63,49 @@ fn shm_format_to_gbm(format: wl_shm::Format) -> Option { } } +#[repr(C)] +struct MetadataCursor { + meta_cursor: spa_sys::spa_meta_cursor, + meta_bitmap: spa_sys::spa_meta_bitmap, + bytes: [u8], +} + +impl MetadataCursor { + pub fn size_of(width: u32, height: u32) -> usize { + std::mem::offset_of!(Self, meta_bitmap) + + std::mem::size_of::() + + width as usize * height as usize * 4 + } + + // , image: &crate::wayland::ShmImage + fn update(&mut self, image: Option<&image::RgbaImage>) { + self.meta_cursor = spa_sys::spa_meta_cursor { + id: 1, + flags: 0, + position: spa_sys::spa_point { x: 50, y: 50 }, + hotspot: spa_sys::spa_point { x: 0, y: 0 }, + //bitmap_offset: 0, + bitmap_offset: std::mem::offset_of!(Self, meta_bitmap) as u32, + }; + let Some(image) = image else { + self.meta_cursor.bitmap_offset = 0; + return; + }; + self.meta_bitmap = spa_sys::spa_meta_bitmap { + format: spa_sys::SPA_VIDEO_FORMAT_RGBA, + size: spa_sys::spa_rectangle { + // XXX + width: image.width(), + height: image.height(), + }, + stride: image.width() as i32 * 4, + offset: std::mem::size_of::() as u32, + }; + // XXX what if buffer is not large enough? + self.bytes[..image.len()].copy_from_slice(image); + } +} + pub struct ScreencastThread { node_id: u32, thread_stop_tx: pipewire::channel::Sender<()>, @@ -104,9 +154,12 @@ struct StreamData { format: gbm::Format, modifier: Option, session: Session, + cursor_stream: Option, + cursor_image: Option, formats: Formats, node_id_tx: Option>>, buffer_damage: HashMap>, + update_cursor: bool, } impl StreamData { @@ -419,6 +472,14 @@ impl StreamData { } fn process(&mut self, stream: &StreamRef) { + if let Some(stream) = &mut self.cursor_stream { + let mut context = Context::from_waker(Waker::noop()); + if let Poll::Ready(image) = stream.poll_next_unpin(&mut context) { + println!("Have cursor image"); + self.cursor_image = image; + } + } + let buffer = unsafe { stream.dequeue_raw_buffer() }; if !buffer.is_null() { let wl_buffer = unsafe { &*((*buffer).user_data as *const wl_buffer::WlBuffer) }; @@ -452,6 +513,18 @@ impl StreamData { } { video_transform.transform = convert_transform(frame.transform); } + if let Some(cursor) = unsafe { + buffer_cursor_find_meta_data( + buffer, + spa_sys::SPA_META_Cursor, + MetadataCursor::size_of(64, 64), + ) + } { + // cursor.update(self.update_cursor); + // XXX update_cursor? + cursor.update(self.cursor_image.as_ref()); + self.update_cursor = false; + } } Err(err) => { if err == WEnum::Value(FailureReason::BufferConstraints) { @@ -491,6 +564,17 @@ fn start_stream( let (node_id_tx, node_id_rx) = oneshot::channel(); let session = wayland_helper.capture_source_session(capture_source, overlay_cursor); + let mut cursor_stream = session.cursor_stream(); + let mut cursor_image = None; + if let Some(stream) = &mut cursor_stream { + let mut context = Context::from_waker(Waker::noop()); + if let Poll::Ready(image) = stream.poll_next_unpin(&mut context) { + println!("Have cursor image 0"); + cursor_image = image; + } + } + + // TODO initial poll? let Some(formats) = block_on(session.wait_for_formats(|formats| formats.clone())) else { return Err(anyhow::anyhow!( @@ -525,11 +609,14 @@ fn start_stream( wayland_helper, dmabuf_helper, session, + cursor_stream, + cursor_image, formats, format: gbm::Format::Abgr8888, modifier: None, node_id_tx: Some(node_id_tx), buffer_damage: HashMap::new(), + update_cursor: true, }; let listener = stream @@ -562,6 +649,28 @@ fn convert_transform(transform: WEnum) -> u32 { } } +// SAFETY: buffer must be non-null, valid as long as return value is used +//unsafe fn buffer_find_meta_data_with_size<'a, T: ?Sized>( +/* +unsafe fn buffer_find_meta_data_with_size<'a, T: ?Sized>( + buffer: *const pipewire_sys::pw_buffer, + type_: u32, + size: usize, +) -> Option<&'a mut T> { + let ptr = spa_sys::spa_buffer_find_meta_data((*buffer).buffer, type_, size); + (std::ptr::slice_from_raw_parts(ptr, size) as *mut T).as_mut() +} +*/ + +unsafe fn buffer_cursor_find_meta_data<'a>( + buffer: *const pipewire_sys::pw_buffer, + type_: u32, + size: usize, +) -> Option<&'a mut MetadataCursor> { + let ptr = spa_sys::spa_buffer_find_meta_data((*buffer).buffer, type_, size); + (std::ptr::slice_from_raw_parts(ptr, size) as *mut MetadataCursor).as_mut() +} + // SAFETY: buffer must be non-null, and valid as long as return value is used unsafe fn buffer_find_meta_data<'a, T>( buffer: *const pipewire_sys::pw_buffer, @@ -618,6 +727,26 @@ fn meta() -> OwnedPod { // TODO: header, video damage } +fn meta_cursor() -> OwnedPod { + OwnedPod::serialize(&pod::Value::Object(pod::Object { + type_: spa_sys::SPA_TYPE_OBJECT_ParamMeta, + id: spa_sys::SPA_PARAM_Meta, + properties: vec![ + pod::Property { + key: spa_sys::SPA_PARAM_META_type, + flags: pod::PropertyFlags::empty(), + value: pod::Value::Id(spa::utils::Id(spa_sys::SPA_META_Cursor)), + }, + pod::Property { + key: spa_sys::SPA_PARAM_META_size, + flags: pod::PropertyFlags::empty(), + // XXX + value: pod::Value::Int(MetadataCursor::size_of(64, 64) as _), + }, + ], + })) +} + fn format_params( dmabuf: Option<&DmabufHelper>, fixated: Option<(gbm::Format, gbm::Modifier)>, @@ -663,6 +792,7 @@ fn other_params(width: u32, height: u32, blocks: u32, allow_dmabuf: bool) -> Vec [ Some(buffers(width, height, blocks, allow_dmabuf)), Some(meta()), + Some(meta_cursor()), ] .into_iter() .flatten() diff --git a/src/wayland/cursor_stream.rs b/src/wayland/cursor_stream.rs new file mode 100644 index 0000000..f9ccc83 --- /dev/null +++ b/src/wayland/cursor_stream.rs @@ -0,0 +1,117 @@ +use cosmic_client_toolkit::screencopy::{CaptureSession, FailureReason, Frame}; +use futures::channel::oneshot; +use std::{ + future::Future, + os::fd::{AsFd, OwnedFd}, + pin::Pin, + sync::Mutex, + task::{Context, Poll}, +}; +use wayland_client::{ + QueueHandle, WEnum, + protocol::{wl_buffer, wl_shm}, +}; + +use super::{AppData, CursorCaptureSessionData, FrameData, WaylandHelper}; +use crate::buffer; + +enum State { + WaitingForFormats, + Capturing(oneshot::Receiver>>), +} + +// TODO wake stream when we get formats? +pub struct CursorStream { + state: Mutex, + // TODO formats + capture_session: CaptureSession, + wayland_helper: WaylandHelper, + // XXX modify pin without mutex? + buffer: Mutex>, +} + +impl CursorStream { + pub(super) fn new(capture_session: &CaptureSession, wayland_helper: &WaylandHelper) -> Self { + Self { + state: Mutex::new(State::WaitingForFormats), + capture_session: capture_session.clone(), + wayland_helper: wayland_helper.clone(), + buffer: Mutex::new(None), + } + } +} + +impl futures::stream::Stream for CursorStream { + type Item = image::RgbaImage; + + fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let data = self + .capture_session + .data::() + .unwrap(); + *data.waker.lock().unwrap() = Some(cx.waker().clone()); + + let mut buffer = self.buffer.lock().unwrap(); + let mut state = self.state.lock().unwrap(); + + if let Some(formats) = &data.formats.lock().unwrap().clone() { + // XXX test if res changed + if buffer + .as_ref() + .is_none_or(|(w, h, _, _)| (*w, *h) != formats.buffer_size) + { + let (width, height) = formats.buffer_size; + let fd = buffer::create_memfd(width, height); + let wl_buffer = self.wayland_helper.create_shm_buffer( + &fd, + width, + height, + width * 4, + wl_shm::Format::Argb8888, + ); + *buffer = Some((width, height, fd, wl_buffer)); + *state = State::WaitingForFormats; // XXX, well, not waiting + } + } + + if let State::Capturing(receiver) = &mut *state { + match std::pin::Pin::new(receiver).poll(cx) { + Poll::Ready(Ok(frame)) => { + // TODO map buffer + let (width, height, fd, _) = &buffer.as_ref().unwrap(); + // XXX unwrap + let mmap = unsafe { memmap2::Mmap::map(fd).unwrap() }; + let mut bytes = mmap.to_vec(); + // Swap BGRA to RGBA + for pixel in bytes.chunks_mut(4) { + pixel.swap(2, 0); + } + let image = image::RgbaImage::from_vec(*width, *height, bytes); + return Poll::Ready(image); + } + // XXX Ignore error + Poll::Ready(Err(_err)) => {} + Poll::Pending => { + return Poll::Pending; + } + } + } + + if let Some((_, _, _, wl_buffer)) = &*buffer { + let (sender, receiver) = oneshot::channel(); + // WIP damage + self.capture_session.capture( + wl_buffer, + &[], + &self.wayland_helper.inner.qh, + FrameData { + frame_data: Default::default(), + sender: Mutex::new(Some(sender)), + }, + ); + *state = State::Capturing(receiver); + } + + Poll::Pending + } +} diff --git a/src/wayland/mod.rs b/src/wayland/mod.rs index bb68d01..27e3006 100644 --- a/src/wayland/mod.rs +++ b/src/wayland/mod.rs @@ -1,14 +1,16 @@ use cosmic_client_toolkit::{ screencopy::{ - CaptureFrame, CaptureOptions, CaptureSession, Capturer, FailureReason, Formats, Frame, - ScreencopyFrameData, ScreencopyFrameDataExt, ScreencopyHandler, ScreencopySessionData, - ScreencopySessionDataExt, ScreencopyState, + CaptureCursorSession, CaptureFrame, CaptureOptions, CaptureSession, Capturer, + FailureReason, Formats, Frame, ScreencopyCursorSessionData, ScreencopyFrameData, + ScreencopyFrameDataExt, ScreencopyHandler, ScreencopySessionData, ScreencopySessionDataExt, + ScreencopyState, }, sctk::{ self, dmabuf::{DmabufFeedback, DmabufFormat, DmabufHandler, DmabufState}, output::{OutputHandler, OutputInfo, OutputState}, registry::{ProvidesRegistryState, RegistryState}, + seat::{self, SeatHandler, SeatState}, shm::{Shm, ShmHandler}, }, toplevel_info::{ToplevelInfo, ToplevelInfoState}, @@ -33,7 +35,7 @@ use std::{ use wayland_client::{ Connection, Dispatch, QueueHandle, WEnum, globals::registry_queue_init, - protocol::{wl_buffer, wl_output, wl_shm, wl_shm_pool}, + protocol::{wl_buffer, wl_output, wl_pointer, wl_seat, wl_shm, wl_shm_pool}, }; use wayland_protocols::{ ext::{ @@ -51,6 +53,8 @@ pub use cosmic_client_toolkit::screencopy::{CaptureSource, Rect}; use crate::buffer; +mod cursor_stream; +pub use cursor_stream::CursorStream; mod gbm_devices; mod toplevel; mod workspaces; @@ -98,6 +102,8 @@ struct WaylandHelperInner { wl_shm: wl_shm::WlShm, dmabuf: Mutex>, zwp_dmabuf: ZwpLinuxDmabufV1, + // TODO: Multiple; remove on seat or cap removal + pointer: Mutex>, } // TODO seperate state object from what is passed to threads @@ -115,6 +121,7 @@ struct AppData { dmabuf_state: DmabufState, toplevel_info_state: ToplevelInfoState, workspace_state: WorkspaceState, + seat_state: SeatState, } impl AppData { @@ -173,6 +180,7 @@ struct SessionState { struct SessionInner { wayland_helper: WaylandHelper, capture_session: CaptureSession, + capture_cursor_session: Option<(CaptureCursorSession, CaptureSession)>, condvar: Condvar, state: Mutex, } @@ -235,6 +243,17 @@ impl Session { // TODO: wait for server to release buffer? receiver.await.unwrap() } + + // XXX Should only be called once + pub fn cursor_stream(&self) -> Option { + let Some((_, capture_session)) = &self.0.capture_cursor_session else { + return None; + }; + Some(cursor_stream::CursorStream::new( + capture_session, + &self.0.wayland_helper, + )) + } } impl WaylandHelper { @@ -258,6 +277,7 @@ impl WaylandHelper { wl_shm: shm_state.wl_shm().clone(), dmabuf: Mutex::new(None), zwp_dmabuf, + pointer: Mutex::new(None), }), }; let dmabuf_state = DmabufState::new(&globals, &qh); @@ -273,6 +293,7 @@ impl WaylandHelper { workspace_state: WorkspaceState::new(®istry_state, &qh), toplevel_info_state: ToplevelInfoState::new(®istry_state, &qh), registry_state, + seat_state: SeatState::new(&globals, &qh), }; event_queue.flush().unwrap(); @@ -375,12 +396,42 @@ impl WaylandHelper { }, ) .unwrap(); + // TODO add only when needed + // TODO one per seat? + // XXX user data + let capture_cursor_session = + self.inner.pointer.lock().unwrap().as_ref().map(|pointer| { + let cursor_session = self + .inner + .capturer + .create_cursor_session( + &source, + pointer, + &self.inner.qh, + ScreencopyCursorSessionData::default(), + ) + .unwrap(); + let capture_session = cursor_session + .capture_session( + &self.inner.qh, + CursorCaptureSessionData { + session: weak_session.clone(), + session_data: ScreencopySessionData::default(), + waker: Mutex::new(None), + formats: Mutex::new(None), + }, + ) + .unwrap(); + (cursor_session, capture_session) + }); + dbg!(&capture_cursor_session); self.inner.conn.flush().unwrap(); SessionInner { wayland_helper: self.clone(), capture_session, + capture_cursor_session, condvar: Condvar::new(), state: Default::default(), } @@ -610,6 +661,13 @@ impl ScreencopyHandler for AppData { session.update(|data| { data.formats = Some(formats.clone()); }); + } else if let Some(data) = session.data::() { + *data.formats.lock().unwrap() = Some(formats.clone()); + let waker = data.waker.lock().unwrap(); + if let Some(waker) = &*waker { + waker.wake_by_ref(); + } + println!("Cursor session formats: {:?}", formats); } } @@ -651,6 +709,17 @@ impl ScreencopyHandler for AppData { let _ = sender.send(Err(reason)); } } + + fn cursor_position( + &mut self, + _conn: &Connection, + _qh: &QueueHandle, + session: &CaptureCursorSession, + x: i32, + y: i32, + ) { + dbg!(x, y); + } } impl DmabufHandler for AppData { @@ -700,6 +769,33 @@ impl DmabufHandler for AppData { } } +impl SeatHandler for AppData { + fn seat_state(&mut self) -> &mut SeatState { + &mut self.seat_state + } + fn new_seat(&mut self, conn: &Connection, qh: &QueueHandle, seat: wl_seat::WlSeat) {} + fn new_capability( + &mut self, + conn: &Connection, + qh: &QueueHandle, + seat: wl_seat::WlSeat, + capability: seat::Capability, + ) { + if capability == seat::Capability::Pointer { + *self.wayland_helper.inner.pointer.lock().unwrap() = Some(seat.get_pointer(qh, ())); + } + } + fn remove_capability( + &mut self, + conn: &Connection, + qh: &QueueHandle, + seat: wl_seat::WlSeat, + capability: seat::Capability, + ) { + } + fn remove_seat(&mut self, conn: &Connection, qh: &QueueHandle, seat: wl_seat::WlSeat) {} +} + impl Dispatch for AppData { fn event( _app_data: &mut Self, @@ -769,6 +865,19 @@ impl ScreencopySessionDataExt for SessionData { } } +struct CursorCaptureSessionData { + session: Weak, + session_data: ScreencopySessionData, + waker: Mutex>, + formats: Mutex>, +} + +impl ScreencopySessionDataExt for CursorCaptureSessionData { + fn screencopy_session_data(&self) -> &ScreencopySessionData { + &self.session_data + } +} + struct FrameData { frame_data: ScreencopyFrameData, #[allow(clippy::type_complexity)] @@ -785,4 +894,6 @@ sctk::delegate_shm!(AppData); sctk::delegate_registry!(AppData); sctk::delegate_output!(AppData); sctk::delegate_dmabuf!(AppData); -cosmic_client_toolkit::delegate_screencopy!(AppData, session: [SessionData], frame: [FrameData]); +sctk::delegate_seat!(AppData); +cosmic_client_toolkit::delegate_screencopy!(AppData); +wayland_client::delegate_noop!(AppData: ignore wl_pointer::WlPointer);