Xwayland integration

[smithay]

Xwayland integration

[psychon]

I have the following diff on top of current master (commit 4d012e1) and it "should be enough". This gets enough things going that I get a working xterm. I did not try any other X11 apps.

The patch (is ugly, but I wanted to get something to work and didn't care about prettyness) (this also only handles the bare minimum necessary events to get Xwayland going; feel free to talk to me some more about X11 WMs if necessary)

Not shown in the diff: I had to override the `wayland-server` dependency via `[patch.crates-io]` so that https://github.com/Smithay/wayland-rs/pull/361 is included.

diff --git a/anvil/Cargo.toml b/anvil/Cargo.toml
index 4dbfd7c1..68d417a6 100644
--- a/anvil/Cargo.toml
+++ b/anvil/Cargo.toml
@@ -15,11 +15,17 @@ slog = { version = "2.1.1" }
 slog-term = "2.3"
 slog-async = "2.2"
 xkbcommon = "0.4.0"
+once_cell = "1.5.2"
 
 [dependencies.smithay]
 path = ".."
 default-features = false
-features = [ "renderer_glium", "backend_egl", "wayland_frontend" ]
+features = [ "renderer_glium", "backend_egl", "wayland_frontend", "xwayland" ]
+
+[dependencies.x11rb]
+default-features = false
+version = "0.7.0"
+features = [ "composite" ]
 
 [build-dependencies]
 gl_generator = "0.14"
diff --git a/anvil/src/main.rs b/anvil/src/main.rs
index f6d28ccd..3340827f 100644
--- a/anvil/src/main.rs
+++ b/anvil/src/main.rs
@@ -25,6 +25,7 @@ mod udev;
 mod window_map;
 #[cfg(feature = "winit")]
 mod winit;
+mod xwayland;
 
 use state::AnvilState;
 
@@ -36,6 +37,8 @@ static POSSIBLE_BACKENDS: &[&str] = &[
 ];
 
 fn main() {
+    let helper = unsafe { smithay::xwayland::LaunchHelper::fork() }.unwrap();
+
     // A logger facility, here we use the terminal here
     let log = slog::Logger::root(
         slog_async::Async::default(slog_term::term_full().fuse()).fuse(),
@@ -50,14 +53,14 @@ fn main() {
         #[cfg(feature = "winit")]
         Some("--winit") => {
             info!(log, "Starting anvil with winit backend");
-            if let Err(()) = winit::run_winit(display, &mut event_loop, log.clone()) {
+            if let Err(()) = winit::run_winit(display, &mut event_loop, helper, log.clone()) {
                 crit!(log, "Failed to initialize winit backend.");
             }
         }
         #[cfg(feature = "udev")]
         Some("--tty-udev") => {
             info!(log, "Starting anvil on a tty using udev");
-            if let Err(()) = udev::run_udev(display, &mut event_loop, log.clone()) {
+            if let Err(()) = udev::run_udev(display, &mut event_loop, helper, log.clone()) {
                 crit!(log, "Failed to initialize tty backend.");
             }
         }
diff --git a/anvil/src/shell.rs b/anvil/src/shell.rs
index f3e9d8ff..5d241a23 100644
--- a/anvil/src/shell.rs
+++ b/anvil/src/shell.rs
@@ -36,11 +36,13 @@ use smithay::{
 use crate::{
     buffer_utils::BufferUtils,
     window_map::{Kind as SurfaceKind, WindowMap},
+    xwayland::X11SurfaceRole,
 };
 
 define_roles!(Roles =>
     [ XdgSurface, XdgSurfaceRole ]
     [ ShellSurface, ShellSurfaceRole]
+    [ X11Surface, X11SurfaceRole ]
     [ DnDIcon, DnDIconRole ]
     [ CursorImage, CursorImageRole ]
 );
@@ -219,6 +221,7 @@ impl PointerGrab for ResizeSurfaceGrab {
                 (self.last_window_size.0 as u32, self.last_window_size.1 as u32),
                 self.edges.into(),
             ),
+            SurfaceKind::X11(_) => eprintln!("XXX Ignoring motion on X11 surface"),
         }
     }
 
@@ -768,6 +771,8 @@ fn surface_commit(
     buffer_utils: &BufferUtils,
     window_map: &RefCell<MyWindowMap>,
 ) {
+    super::xwayland::commit_hook(surface);
+
     let mut geometry = None;
     let mut min_size = (0, 0);
     let mut max_size = (0, 0);
diff --git a/anvil/src/state.rs b/anvil/src/state.rs
index 48bc6ce8..eac4cdf7 100644
--- a/anvil/src/state.rs
+++ b/anvil/src/state.rs
@@ -21,6 +21,7 @@ use smithay::{
         seat::{CursorImageStatus, KeyboardHandle, PointerHandle, Seat, XkbConfig},
         shm::init_shm_global,
     },
+    xwayland::{LaunchHelper, XWayland},
 };
 
 #[cfg(feature = "udev")]
@@ -28,7 +29,7 @@ use smithay::backend::session::{auto::AutoSession, Session};
 
 #[cfg(feature = "udev")]
 use crate::udev::MyOutput;
-use crate::{buffer_utils::BufferUtils, shell::init_shell};
+use crate::{buffer_utils::BufferUtils, shell::init_shell, xwayland::XWm};
 
 pub struct AnvilState {
     pub socket_name: String,
@@ -51,6 +52,7 @@ pub struct AnvilState {
     pub session: Option<AutoSession>,
     // things we must keep alive
     _wayland_event_source: Source<Generic<Fd>>,
+    xwayland: Option<XWayland<XWm>>,
 }
 
 impl AnvilState {
@@ -62,6 +64,7 @@ impl AnvilState {
         #[cfg(not(feature = "udev"))] _session: Option<()>,
         #[cfg(feature = "udev")] output_map: Option<Rc<RefCell<Vec<MyOutput>>>>,
         #[cfg(not(feature = "udev"))] _output_map: Option<()>,
+        helper: LaunchHelper,
         log: slog::Logger,
     ) -> AnvilState {
         // init the wayland connection
@@ -153,14 +156,14 @@ impl AnvilState {
             })
             .expect("Failed to initialize the keyboard");
 
-        AnvilState {
+        let mut state = AnvilState {
             running: Arc::new(AtomicBool::new(true)),
-            display,
-            handle,
+            display: display.clone(),
+            handle: handle.clone(),
             ctoken: shell_handles.token,
-            window_map: shell_handles.window_map,
+            window_map: shell_handles.window_map.clone(),
             dnd_icon,
-            log,
+            log: log.clone(),
             socket_name,
             pointer,
             keyboard,
@@ -171,7 +174,15 @@ impl AnvilState {
             seat_name,
             #[cfg(feature = "udev")]
             session,
+            xwayland: None,
             _wayland_event_source,
-        }
+        };
+
+        info!(log, "Hey Uli: Starting XWayland");
+        let wm = XWm::new(handle.clone(), shell_handles.token, shell_handles.window_map);
+        let xwl = XWayland::init(wm, handle.clone(), display.clone(), &mut state, log.clone(), helper).unwrap();
+        state.xwayland = Some(xwl);
+
+        state
     }
 }
diff --git a/anvil/src/udev.rs b/anvil/src/udev.rs
index e5a8b596..279938c6 100644
--- a/anvil/src/udev.rs
+++ b/anvil/src/udev.rs
@@ -60,6 +60,7 @@ use smithay::{
         output::{Mode, Output, PhysicalProperties},
         seat::CursorImageStatus,
     },
+    xwayland::LaunchHelper,
 };
 
 use crate::buffer_utils::BufferUtils;
@@ -93,6 +94,7 @@ type RenderSurface = FallbackSurface<
 pub fn run_udev(
     display: Rc<RefCell<Display>>,
     event_loop: &mut EventLoop<AnvilState>,
+    helper: LaunchHelper,
     log: Logger,
 ) -> Result<(), ()> {
     let name = display
@@ -129,6 +131,7 @@ pub fn run_udev(
         buffer_utils.clone(),
         Some(session),
         Some(output_map.clone()),
+        helper,
         log.clone(),
     );
 
diff --git a/anvil/src/window_map.rs b/anvil/src/window_map.rs
index 54eb33f7..1abadd4a 100644
--- a/anvil/src/window_map.rs
+++ b/anvil/src/window_map.rs
@@ -12,11 +12,15 @@ use smithay::{
     },
 };
 
-use crate::shell::SurfaceData;
+use crate::{
+    shell::SurfaceData,
+    xwayland::X11Surface,
+};
 
 pub enum Kind<R> {
     Xdg(ToplevelSurface<R>),
     Wl(ShellSurface<R>),
+    X11(X11Surface<R>),
 }
 
 // We implement Clone manually because #[derive(..)] would require R: Clone.
@@ -25,6 +29,7 @@ impl<R> Clone for Kind<R> {
         match self {
             Kind::Xdg(xdg) => Kind::Xdg(xdg.clone()),
             Kind::Wl(wl) => Kind::Wl(wl.clone()),
+            Kind::X11(x11) => Kind::X11(x11.clone()),
         }
     }
 }
@@ -37,12 +42,14 @@ where
         match *self {
             Kind::Xdg(ref t) => t.alive(),
             Kind::Wl(ref t) => t.alive(),
+            Kind::X11(ref t) => t.alive(),
         }
     }
     pub fn get_surface(&self) -> Option<&wl_surface::WlSurface> {
         match *self {
             Kind::Xdg(ref t) => t.get_surface(),
             Kind::Wl(ref t) => t.get_surface(),
+            Kind::X11(ref t) => t.get_surface(),
         }
     }
 
@@ -51,6 +58,7 @@ where
         match (self, other) {
             (Kind::Xdg(a), Kind::Xdg(b)) => a.equals(b),
             (Kind::Wl(a), Kind::Wl(b)) => a.equals(b),
+            (Kind::X11(a), Kind::X11(b)) => a.equals(b),
             _ => false,
         }
     }
diff --git a/anvil/src/winit.rs b/anvil/src/winit.rs
index eafbfcf2..9b3f4184 100644
--- a/anvil/src/winit.rs
+++ b/anvil/src/winit.rs
@@ -13,6 +13,7 @@ use smithay::{
         output::{Mode, Output, PhysicalProperties},
         seat::CursorImageStatus,
     },
+    xwayland::LaunchHelper
 };
 
 use slog::Logger;
@@ -24,6 +25,7 @@ use crate::state::AnvilState;
 pub fn run_winit(
     display: Rc<RefCell<Display>>,
     event_loop: &mut EventLoop<AnvilState>,
+    helper: LaunchHelper,
     log: Logger,
 ) -> Result<(), ()> {
     let (renderer, mut input) = winit::init(log.clone()).map_err(|err| {
@@ -58,6 +60,7 @@ pub fn run_winit(
         buffer_utils,
         None,
         None,
+        helper,
         log.clone(),
     );
 
diff --git a/anvil/src/xwayland.rs b/anvil/src/xwayland.rs
new file mode 100644
index 00000000..4ca13262
--- /dev/null
+++ b/anvil/src/xwayland.rs
@@ -0,0 +1,285 @@
+use std::{
+    cell::RefCell,
+    collections::HashMap,
+    convert::TryFrom,
+    rc::Rc,
+    sync::Mutex,
+    os::unix::{net::UnixStream, io::AsRawFd},
+};
+
+use smithay::{
+    reexports::{
+        calloop::{
+            generic::{Fd, Generic},
+            Interest, LoopHandle, Mode, Source,
+        },
+        wayland_server::{protocol::wl_surface::WlSurface, Client},
+    },
+    wayland::compositor::CompositorToken,
+    xwayland::{XWindowManager},
+};
+
+use x11rb::{
+    connection::Connection as _,
+    rust_connection::{RustConnection, DefaultStream},
+    protocol::{
+        composite::{ConnectionExt as _, Redirect},
+        xproto::{
+            ConnectionExt as _,
+            EventMask,
+            ChangeWindowAttributesAux,
+            ConfigWindow,
+            ConfigureWindowAux,
+            Window,
+            WindowClass,
+        },
+    },
+};
+
+use crate::{
+    state::AnvilState,
+    shell::{MyWindowMap, Roles},
+    window_map::Kind,
+};
+
+x11rb::atom_manager! {
+    Atoms: AtomsCookie {
+        WL_SURFACE_ID,
+    }
+}
+
+struct XWmInner {
+    connection: RustConnection,
+    atoms: Atoms,
+    _source: Source<Generic<Fd>>,
+    token: CompositorToken<Roles>,
+    unpaired_surfaces: HashMap<u32, Window>,
+    paired_surfaces: Vec<(Window, WlSurface)>,
+    window_map: Rc<RefCell<MyWindowMap>>,
+}
+
+impl XWmInner {
+    fn new_window(&mut self, window: Window, surface: &WlSurface) {
+        self.paired_surfaces.push((window, surface.clone()));
+        println!("matched X11 window {:#x} and {:?}", window, surface);
+        assert!(self.token.give_role_with(surface, X11SurfaceRole { window }).is_ok());
+        let x11surface = X11Surface {
+            window,
+            wl_surface: surface.clone(),
+            token: self.token,
+        };
+        let _ = x11surface.window; // Hack to silence a compiler warning; just remove this line
+        self.token.with_role_data::<X11SurfaceRole, _, _>(surface, |r| { let _ = r.window; }).unwrap(); // Hack to silence a compiler warning; just remove this line
+        self.window_map.borrow_mut().insert(Kind::X11(x11surface), (0, 0));
+    }
+}
+
+use once_cell::sync::OnceCell;
+static XWAYLAND_CLIENT: OnceCell<Mutex<Option<Client>>> = OnceCell::new();
+
+pub struct XWm {
+    handle: LoopHandle<AnvilState>,
+    token: CompositorToken<Roles>,
+    window_map: Rc<RefCell<MyWindowMap>>,
+}
+
+impl XWm {
+    pub fn new(handle: LoopHandle<AnvilState>, token: CompositorToken<Roles>, window_map: Rc<RefCell<MyWindowMap>>) -> Self {
+        Self { handle, token, window_map }
+    }
+}
+
+impl XWindowManager for XWm {
+    fn xwayland_ready(&mut self, connection: UnixStream, client: Client) {
+        println!("Hey Uli: XWayland is ready, DISPLAY={}", std::env::var("DISPLAY").unwrap_or("NONE".to_string()));
+
+        let (connection, atoms) = start_wm(connection).unwrap();
+        let fd = connection.stream().as_raw_fd();
+        let source = self.handle.insert_source(
+            Generic::new(Fd(fd), Interest::Readable, Mode::Level),
+            |_, _, _| {
+                let guard = XWAYLAND_CLIENT.get().unwrap().lock().unwrap();
+                let client = guard.as_ref().unwrap();
+                let inner = client.data_map().get::<RefCell<XWmInner>>().unwrap();
+                read_events(client, &mut *inner.borrow_mut()).unwrap();
+                Ok(())
+            },
+        ).unwrap();
+        let inner = XWmInner {
+            connection,
+            atoms,
+            _source: source,
+            token: self.token,
+            window_map: self.window_map.clone(),
+            unpaired_surfaces: Default::default(),
+            paired_surfaces: Default::default(),
+        };
+        client.data_map().insert_if_missing(|| RefCell::new(inner));
+        let mutex = XWAYLAND_CLIENT.get_or_init(|| Mutex::new(None));
+        *mutex.lock().unwrap() = Some(client);
+
+        // The root window gets a ClientMessageEvent of type WL_SURFACE_ID with its first 32 bit
+        // member set to the wl surface id of the X11 window available in its window member. We
+        // have to use that to build a mapping between X11 windows and WL surfaces.
+        // However, there is a race: Either the ClientMessage or the surface creation arrives
+        // first, so we have to be prepared for both orders.
+        // At this point, we assign the surface some special role and make it visible.
+
+        // Everyone seems to keep a list of unpaired windows and use that for the above.
+    }
+
+    fn xwayland_exited(&mut self) {
+        println!("Hey Uli: XWayland exited");
+    }
+}
+
+pub(crate) fn commit_hook(surface: &WlSurface) {
+    if let Some(client) = surface.as_ref().client() {
+        if let Some(inner) = client.data_map().get::<RefCell<XWmInner>>() {
+            let mut inner = inner.borrow_mut();
+            if let Some(window) = inner.unpaired_surfaces.remove(&surface.as_ref().id()) {
+                inner.new_window(window, surface);
+            }
+        }
+    }
+}
+
+fn start_wm(connection: UnixStream) -> Result<(RustConnection, Atoms), Box<dyn std::error::Error>> {
+    // It's always screen 0 with Xwayland
+    let screen = 0;
+    let stream = DefaultStream::from_unix_stream(connection)?;
+    let connection = RustConnection::connect_to_stream(stream, screen)?;
+    let atoms = Atoms::new(&connection)?;
+
+    let screen = &connection.setup().roots[0];
+
+    // This is how we actually become the WM.
+    connection.change_window_attributes(
+        screen.root,
+        &ChangeWindowAttributesAux::default().event_mask(EventMask::SubstructureRedirect),
+    )?;
+
+    // Xwayland waits until the WM_S0 selection is acquired before it starts working
+    let win = connection.generate_id()?;
+    connection.create_window(
+        screen.root_depth,
+        win,
+        screen.root,
+        // x, y, width, height, border width
+        0,
+        0,
+        1,
+        1,
+        0,
+        WindowClass::InputOutput,
+        x11rb::COPY_FROM_PARENT,
+        &Default::default(),
+    )?;
+    let wm_s0 = connection.intern_atom(false, b"WM_S0")?.reply()?;
+
+    // Screw ICCCM, we know that we are alone and do not have to do this properly
+    connection.set_selection_owner(win, wm_s0.atom, x11rb::CURRENT_TIME)?;
+
+    // No idea why I am doing this, but Xwayland seems to require it
+    connection.composite_redirect_subwindows(screen.root, Redirect::Manual)?;
+
+    let atoms = atoms.reply()?;
+
+    // Ensure all our requests are sent
+    connection.flush()?;
+
+    Ok((connection, atoms))
+}
+
+fn read_events(client: &Client, inner: &mut XWmInner) -> Result<(), Box<dyn std::error::Error>> {
+    use x11rb::protocol::Event;
+
+    while let Some(event) = inner.connection.poll_for_event()? {
+        eprintln!("X11: Got event {:?}", event);
+        match event {
+            Event::ConfigureRequest(r) => {
+                // Just grant the wish
+                let mut aux = ConfigureWindowAux::default();
+                if r.value_mask & u16::from(ConfigWindow::StackMode) != 0 {
+                    aux = aux.stack_mode(r.stack_mode);
+                }
+                if r.value_mask & u16::from(ConfigWindow::Sibling) != 0 {
+                    aux = aux.sibling(r.sibling);
+                }
+                if r.value_mask & u16::from(ConfigWindow::X) != 0 {
+                    aux = aux.x(i32::try_from(r.x).unwrap());
+                }
+                if r.value_mask & u16::from(ConfigWindow::Y) != 0 {
+                    aux = aux.y(i32::try_from(r.y).unwrap());
+                }
+                if r.value_mask & u16::from(ConfigWindow::Width) != 0 {
+                    aux = aux.width(u32::try_from(r.width).unwrap());
+                }
+                if r.value_mask & u16::from(ConfigWindow::Height) != 0 {
+                    aux = aux.height(u32::try_from(r.height).unwrap());
+                }
+                if r.value_mask & u16::from(ConfigWindow::BorderWidth) != 0 {
+                    aux = aux.border_width(u32::try_from(r.border_width).unwrap());
+                }
+                inner.connection.configure_window(r.window, &aux)?;
+            }
+            Event::MapRequest(r) => {
+                // Just grant the wish
+                inner.connection.map_window(r.window)?;
+            }
+            Event::ClientMessage(msg) => {
+                if msg.type_ == inner.atoms.WL_SURFACE_ID {
+                    let id = msg.data.as_data32()[0];
+                    let surf = client.get_resource::<WlSurface>(id);
+                    match surf {
+                        None => {
+                            inner.unpaired_surfaces.insert(id, msg.window);
+                        }
+                        Some(surf) => inner.new_window(msg.window, &surf),
+                    }
+                }
+            }
+            _ => {}
+        }
+    }
+    inner.connection.flush()?;
+    Ok(())
+}
+
+pub struct X11SurfaceRole {
+    window: Window,
+}
+
+pub struct X11Surface<R> {
+    window: Window,
+    wl_surface: WlSurface,
+    token: CompositorToken<R>
+}
+
+impl<R> Clone for X11Surface<R> {
+    fn clone(&self) -> Self {
+        Self {
+            window: self.window,
+            wl_surface: self.wl_surface.clone(),
+            token: self.token,
+        }
+    }
+}
+
+impl<R> X11Surface<R> {
+    pub fn alive(&self) -> bool {
+        self.wl_surface.as_ref().is_alive()
+    }
+
+    pub fn equals(&self, other: &Self) -> bool {
+        self.alive() && other.alive() && self.wl_surface.as_ref().equals(&other.wl_surface.as_ref())
+    }
+
+    pub fn get_surface(&self) -> Option<&WlSurface> {
+        if self.alive() {
+            Some(&self.wl_surface)
+        } else {
+            None
+        }
+    }
+}
diff --git a/src/xwayland/xserver.rs b/src/xwayland/xserver.rs
index 42186965..dce9aedf 100644
--- a/src/xwayland/xserver.rs
+++ b/src/xwayland/xserver.rs
@@ -330,6 +330,7 @@ pub(crate) fn exec_xwayland(
     }
     // the WAYLAND_SOCKET var tells XWayland where to connect as a wayland client
     env::set_var("WAYLAND_SOCKET", format!("{}", wayland_socket.as_raw_fd()));
+    env::set_var("WAYLAND_DEBUG", "client");
 
     // ignore SIGUSR1, this will make the XWayland server send us this
     // signal when it is ready apparently

[elinorbgr]

We now have the core mechanism handling the XWayland server iteself. "Only" remains writing some WM utilities / integration.

[DemiMarie]

Is this now fixed?

[PolyMeilex]

Mostly yes, I'm using smithay's xwayland utils and I haven't had any problems so far, I think that there are some helpers still planed, but all of the groundwork is already laid out.