xwayland: add option to enable/disable xwayland (#7633)

* config: add xwayland enabled option to config

* xwayland: use DISPLAY env variable for enable/disable of new launches

* xwayland: close X11 windows when turning of XWayland

* clang: format fix

* config: add better description for xwayland:enabled

* xwayland: close X11 windows on disable without crashes

* xwayland: better method of informing CXWayland if xwayland enabled

* xwayland: prevent closing non-xwayland windows on disable

* misc: loop formatting

8 files changed, 48 insertions, 6 deletions

diff --git a/src/Compositor.cpp b/src/Compositor.cpp
index ba9ff96b..da659654 100644
--- a/src/Compositor.cpp
+++ b/src/Compositor.cpp
@@ -615,7 +615,7 @@ void CCompositor::initManagers(eManagersInitStage stage) {
             g_pCursorManager = std::make_unique<CCursorManager>();
 
             Debug::log(LOG, "Starting XWayland");
-            g_pXWayland = std::make_unique<CXWayland>();
+            g_pXWayland = std::make_unique<CXWayland>(g_pCompositor->m_bEnableXwayland);
         } break;
         default: UNREACHABLE();
     }
diff --git a/src/Compositor.hpp b/src/Compositor.hpp
index bb986f5e..f17d86e5 100644
--- a/src/Compositor.hpp
+++ b/src/Compositor.hpp
@@ -92,6 +92,7 @@ class CCompositor {
     CMonitor*                                  m_pUnsafeOutput   = nullptr; // fallback output for the unsafe state
     bool                                       m_bIsShuttingDown = false;
     bool                                       m_bDesktopEnvSet  = false;
+    bool                                       m_bEnableXwayland = true;
 
     // ------------------------------------------------- //
 
diff --git a/src/config/ConfigDescriptions.hpp b/src/config/ConfigDescriptions.hpp
index 73f3c9a3..00ebde6a 100644
--- a/src/config/ConfigDescriptions.hpp
+++ b/src/config/ConfigDescriptions.hpp
@@ -1116,6 +1116,12 @@ inline static const std::vector<SConfigOptionDescription> CONFIG_OPTIONS = {
      */
 
     SConfigOptionDescription{
+        .value       = "xwayland:enabled",
+        .description = "allow running applications using X11",
+        .type        = CONFIG_OPTION_BOOL,
+        .data        = SConfigOptionDescription::SBoolData{true},
+    },
+    SConfigOptionDescription{
         .value       = "xwayland:use_nearest_neighbor",
         .description = "uses the nearest neighbor filtering for xwayland apps, making them pixelated rather than blurry",
         .type        = CONFIG_OPTION_BOOL,
diff --git a/src/config/ConfigManager.cpp b/src/config/ConfigManager.cpp
index 5940a6fe..740dbbdf 100644
--- a/src/config/ConfigManager.cpp
+++ b/src/config/ConfigManager.cpp
@@ -6,6 +6,7 @@
 #include "config/ConfigValue.hpp"
 #include "helpers/varlist/VarList.hpp"
 #include "../protocols/LayerShell.hpp"
+#include "../xwayland/XWayland.hpp"
 
 #include <cstddef>
 #include <cstdint>
@@ -523,6 +524,7 @@ CConfigManager::CConfigManager() {
     m_pConfig->addConfigValue("gestures:workspace_swipe_touch", Hyprlang::INT{0});
     m_pConfig->addConfigValue("gestures:workspace_swipe_touch_invert", Hyprlang::INT{0});
 
+    m_pConfig->addConfigValue("xwayland:enabled", Hyprlang::INT{1});
     m_pConfig->addConfigValue("xwayland:use_nearest_neighbor", Hyprlang::INT{1});
     m_pConfig->addConfigValue("xwayland:force_zero_scaling", Hyprlang::INT{0});
 
@@ -860,6 +862,29 @@ void CConfigManager::postConfigReload(const Hyprlang::CParseResult& result) {
         ensureVRR();
     }
 
+#ifndef NO_XWAYLAND
+    const auto PENABLEXWAYLAND = std::any_cast<Hyprlang::INT>(m_pConfig->getConfigValue("xwayland:enabled"));
+    // enable/disable xwayland usage
+    if (!isFirstLaunch) {
+        bool prevEnabledXwayland = g_pCompositor->m_bEnableXwayland;
+        if (PENABLEXWAYLAND != prevEnabledXwayland) {
+            g_pCompositor->m_bEnableXwayland = PENABLEXWAYLAND;
+            if (PENABLEXWAYLAND) {
+                Debug::log(LOG, "xwayland has been enabled");
+            } else {
+                Debug::log(LOG, "xwayland has been disabled, cleaning up...");
+                for (auto& w : g_pCompositor->m_vWindows) {
+                    if (w->m_pXDGSurface || !w->m_bIsX11)
+                        continue;
+                    g_pCompositor->closeWindow(w);
+                }
+            }
+            g_pXWayland = std::make_unique<CXWayland>(g_pCompositor->m_bEnableXwayland);
+        }
+    } else
+        g_pCompositor->m_bEnableXwayland = PENABLEXWAYLAND;
+#endif
+
     if (!isFirstLaunch && !g_pCompositor->m_bUnsafeState)
         refreshGroupBarGradients();
 
diff --git a/src/xwayland/Server.cpp b/src/xwayland/Server.cpp
index 200bec70..f3bf5768 100644
--- a/src/xwayland/Server.cpp
+++ b/src/xwayland/Server.cpp
@@ -432,7 +432,8 @@ int CXWaylandServer::ready(int fd, uint32_t mask) {
     pipeSource = nullptr;
 
     // start the wm
-    g_pXWayland->pWM = std::make_unique<CXWM>();
+    if (!g_pXWayland->pWM)
+        g_pXWayland->pWM = std::make_unique<CXWM>();
 
     g_pCursorManager->setXWaylandCursor();
 
diff --git a/src/xwayland/XWM.cpp b/src/xwayland/XWM.cpp
index cb4d8f4d..e8e2258a 100644
--- a/src/xwayland/XWM.cpp
+++ b/src/xwayland/XWM.cpp
@@ -890,6 +890,10 @@ CXWM::~CXWM() {
 
     if (eventSource)
         wl_event_source_remove(eventSource);
+
+    for (auto const& sr : surfaces) {
+        sr->events.destroy.emit();
+    }
 }
 
 void CXWM::setActiveWindow(xcb_window_t window) {
diff --git a/src/xwayland/XWayland.cpp b/src/xwayland/XWayland.cpp
index 8d45fa63..8cdb1fca 100644
--- a/src/xwayland/XWayland.cpp
+++ b/src/xwayland/XWayland.cpp
@@ -1,12 +1,17 @@
 #include "XWayland.hpp"
 #include "../debug/Log.hpp"
 
-CXWayland::CXWayland() {
+CXWayland::CXWayland(const bool enabled) {
 #ifndef NO_XWAYLAND
     Debug::log(LOG, "Starting up the XWayland server");
 
     pServer = std::make_unique<CXWaylandServer>();
 
+    if (!enabled) {
+        unsetenv("DISPLAY");
+        return;
+    }
+
     if (!pServer->create()) {
         Debug::log(ERR, "XWayland failed to start: it will not work.");
         return;
@@ -25,4 +30,4 @@ void CXWayland::setCursor(unsigned char* pixData, uint32_t stride, const Vector2
 
     pWM->setCursor(pixData, stride, size, hotspot);
 #endif
-}
\ No newline at end of file
+}
diff --git a/src/xwayland/XWayland.hpp b/src/xwayland/XWayland.hpp
index 40c0ba65..96253d19 100644
--- a/src/xwayland/XWayland.hpp
+++ b/src/xwayland/XWayland.hpp
@@ -17,7 +17,7 @@ class CXWM;
 
 class CXWayland {
   public:
-    CXWayland();
+    CXWayland(const bool enabled);
 
 #ifndef NO_XWAYLAND
     std::unique_ptr<CXWaylandServer> pServer;
@@ -126,4 +126,4 @@ inline std::unordered_map<std::string, uint32_t> HYPRATOMS = {
     HYPRATOM("DELETE"),
     HYPRATOM("TEXT"),
     HYPRATOM("INCR"),
-};
\ No newline at end of file
+};