X11: Disable maximize on non-resizable windows (#1000)

* X11: Disable maximize on non-resizable windows

* Add a note for the source for Motif WM constants

CHANGELOG.md

@@ -1,5 +1,6 @@
 # Unreleased
 
+- On X11, non-resizable windows now have maximize explicitly disabled.
 - On Windows, support paths longer than MAX_PATH (260 characters) in `WindowEvent::DroppedFile`
 and `WindowEvent::HoveredFile`.
 - On Mac, implement `DeviceEvent::Button`.

src/platform_impl/linux/x11/util/hint.rs

@@ -1,9 +1,8 @@
+use std::slice;
 use std::sync::Arc;
 
 use super::*;
 
-pub const MWM_HINTS_DECORATIONS: c_ulong = 2;
-
 #[derive(Debug)]
 pub enum StateOperation {
     Remove = 0, // _NET_WM_STATE_REMOVE
@@ -93,6 +92,92 @@ impl WindowType {
     }
 }
 
+pub struct MotifHints {
+    hints: MwmHints,
+}
+
+#[repr(C)]
+struct MwmHints {
+    flags: c_ulong,
+    functions: c_ulong,
+    decorations: c_ulong,
+    input_mode: c_long,
+    status: c_ulong,
+}
+
+#[allow(dead_code)]
+mod mwm {
+    use libc::c_ulong;
+
+    // Motif WM hints are obsolete, but still widely supported.
+    // https://stackoverflow.com/a/1909708
+    pub const MWM_HINTS_FUNCTIONS: c_ulong = 1 << 0;
+    pub const MWM_HINTS_DECORATIONS: c_ulong = 1 << 1;
+
+    pub const MWM_FUNC_ALL: c_ulong = 1 << 0;
+    pub const MWM_FUNC_RESIZE: c_ulong = 1 << 1;
+    pub const MWM_FUNC_MOVE: c_ulong = 1 << 2;
+    pub const MWM_FUNC_MINIMIZE: c_ulong = 1 << 3;
+    pub const MWM_FUNC_MAXIMIZE: c_ulong = 1 << 4;
+    pub const MWM_FUNC_CLOSE: c_ulong = 1 << 5;
+}
+
+impl MotifHints {
+    pub fn new() -> MotifHints {
+        MotifHints {
+            hints: MwmHints {
+                flags: 0,
+                functions: 0,
+                decorations: 0,
+                input_mode: 0,
+                status: 0,
+            },
+        }
+    }
+
+    pub fn set_decorations(&mut self, decorations: bool) {
+        self.hints.flags |= mwm::MWM_HINTS_DECORATIONS;
+        self.hints.decorations = decorations as c_ulong;
+    }
+
+    pub fn set_maximizable(&mut self, maximizable: bool) {
+        if maximizable {
+            self.add_func(mwm::MWM_FUNC_MAXIMIZE);
+        } else {
+            self.remove_func(mwm::MWM_FUNC_MAXIMIZE);
+        }
+    }
+
+    fn add_func(&mut self, func: c_ulong) {
+        if self.hints.flags & mwm::MWM_HINTS_FUNCTIONS != 0 {
+            if self.hints.functions & mwm::MWM_FUNC_ALL != 0 {
+                self.hints.functions &= !func;
+            } else {
+                self.hints.functions |= func;
+            }
+        }
+    }
+
+    fn remove_func(&mut self, func: c_ulong) {
+        if self.hints.flags & mwm::MWM_HINTS_FUNCTIONS == 0 {
+            self.hints.flags |= mwm::MWM_HINTS_FUNCTIONS;
+            self.hints.functions = mwm::MWM_FUNC_ALL;
+        }
+
+        if self.hints.functions & mwm::MWM_FUNC_ALL != 0 {
+            self.hints.functions |= func;
+        } else {
+            self.hints.functions &= !func;
+        }
+    }
+}
+
+impl MwmHints {
+    fn as_slice(&self) -> &[c_ulong] {
+        unsafe { slice::from_raw_parts(self as *const _ as *const c_ulong, 5) }
+    }
+}
+
 pub struct NormalHints<'a> {
     size_hints: XSmartPointer<'a, ffi::XSizeHints>,
 }
@@ -254,4 +339,32 @@ impl XConnection {
         }
         Flusher::new(self)
     }
+
+    pub fn get_motif_hints(&self, window: ffi::Window) -> MotifHints {
+        let motif_hints = unsafe { self.get_atom_unchecked(b"_MOTIF_WM_HINTS\0") };
+
+        let mut hints = MotifHints::new();
+
+        if let Ok(props) = self.get_property::<c_ulong>(window, motif_hints, motif_hints) {
+            hints.hints.flags = props.get(0).cloned().unwrap_or(0);
+            hints.hints.functions = props.get(1).cloned().unwrap_or(0);
+            hints.hints.decorations = props.get(2).cloned().unwrap_or(0);
+            hints.hints.input_mode = props.get(3).cloned().unwrap_or(0) as c_long;
+            hints.hints.status = props.get(4).cloned().unwrap_or(0);
+        }
+
+        hints
+    }
+
+    pub fn set_motif_hints(&self, window: ffi::Window, hints: &MotifHints) -> Flusher<'_> {
+        let motif_hints = unsafe { self.get_atom_unchecked(b"_MOTIF_WM_HINTS\0") };
+
+        self.change_property(
+            window,
+            motif_hints,
+            motif_hints,
+            PropMode::Replace,
+            hints.hints.as_slice(),
+        )
+    }
 }

src/platform_impl/linux/x11/window.rs

@@ -667,20 +667,11 @@ impl UnownedWindow {
     }
 
     fn set_decorations_inner(&self, decorations: bool) -> util::Flusher<'_> {
-        let wm_hints = unsafe { self.xconn.get_atom_unchecked(b"_MOTIF_WM_HINTS\0") };
-        self.xconn.change_property(
-            self.xwindow,
-            wm_hints,
-            wm_hints,
-            util::PropMode::Replace,
-            &[
-                util::MWM_HINTS_DECORATIONS, // flags
-                0,                           // functions
-                decorations as c_ulong,      // decorations
-                0,                           // input mode
-                0,                           // status
-            ],
-        )
+        let mut hints = self.xconn.get_motif_hints(self.xwindow);
+
+        hints.set_decorations(decorations);
+
+        self.xconn.set_motif_hints(self.xwindow, &hints)
     }
 
     #[inline]
@@ -691,6 +682,14 @@ impl UnownedWindow {
         self.invalidate_cached_frame_extents();
     }
 
+    fn set_maximizable_inner(&self, maximizable: bool) -> util::Flusher<'_> {
+        let mut hints = self.xconn.get_motif_hints(self.xwindow);
+
+        hints.set_maximizable(maximizable);
+
+        self.xconn.set_motif_hints(self.xwindow, &hints)
+    }
+
     fn set_always_on_top_inner(&self, always_on_top: bool) -> util::Flusher<'_> {
         let above_atom = unsafe { self.xconn.get_atom_unchecked(b"_NET_WM_STATE_ABOVE\0") };
         self.set_netwm(always_on_top.into(), (above_atom as c_long, 0, 0, 0))
@@ -985,6 +984,8 @@ impl UnownedWindow {
             (window_size.clone(), window_size)
         };
 
+        self.set_maximizable_inner(resizable).queue();
+
         let dpi_factor = self.hidpi_factor();
         let min_inner_size = logical_min
             .map(|logical_size| logical_size.to_physical(dpi_factor))