core: avoid locking 2 wayland sockets (#6971)

* fix: avoid locking 2 wayland sockets

* format

* fix formatting

3 files changed, 24 insertions, 53 deletions

diff --git a/src/Compositor.cpp b/src/Compositor.cpp
index 02ca5363..e3ef664d 100644
--- a/src/Compositor.cpp
+++ b/src/Compositor.cpp
@@ -207,7 +207,7 @@ void CCompositor::setRandomSplash() {
 static std::vector<SP<Aquamarine::IOutput>> pendingOutputs;
 
 //
-void CCompositor::initServer() {
+void CCompositor::initServer(std::string socketName, int socketFd) {
 
     m_sWLDisplay = wl_display_create();
 
@@ -267,16 +267,25 @@ void CCompositor::initServer() {
     m_iDRMFD = m_pAqBackend->drmFD();
     Debug::log(LOG, "Running on DRMFD: {}", m_iDRMFD);
 
-    // get socket, avoid using 0
-    for (int candidate = 1; candidate <= 32; candidate++) {
-        const auto CANDIDATESTR = ("wayland-" + std::to_string(candidate));
-        const auto RETVAL       = wl_display_add_socket(m_sWLDisplay, CANDIDATESTR.c_str());
+    if (!socketName.empty() && socketFd != -1) {
+        fcntl(socketFd, F_SETFD, FD_CLOEXEC);
+        const auto RETVAL = wl_display_add_socket_fd(m_sWLDisplay, socketFd);
         if (RETVAL >= 0) {
-            m_szWLDisplaySocket = CANDIDATESTR;
-            Debug::log(LOG, "wl_display_add_socket for {} succeeded with {}", CANDIDATESTR, RETVAL);
-            break;
-        } else {
-            Debug::log(WARN, "wl_display_add_socket for {} returned {}: skipping candidate {}", CANDIDATESTR, RETVAL, candidate);
+            m_szWLDisplaySocket = socketName;
+            Debug::log(LOG, "wl_display_add_socket_fd for {} succeeded with {}", socketName, RETVAL);
+        } else
+            Debug::log(WARN, "wl_display_add_socket_fd for {} returned {}: skipping", socketName, RETVAL);
+    } else {
+        // get socket, avoid using 0
+        for (int candidate = 1; candidate <= 32; candidate++) {
+            const auto CANDIDATESTR = ("wayland-" + std::to_string(candidate));
+            const auto RETVAL       = wl_display_add_socket(m_sWLDisplay, CANDIDATESTR.c_str());
+            if (RETVAL >= 0) {
+                m_szWLDisplaySocket = CANDIDATESTR;
+                Debug::log(LOG, "wl_display_add_socket for {} succeeded with {}", CANDIDATESTR, RETVAL);
+                break;
+            } else
+                Debug::log(WARN, "wl_display_add_socket for {} returned {}: skipping candidate {}", CANDIDATESTR, RETVAL, candidate);
         }
     }
 
@@ -292,7 +301,6 @@ void CCompositor::initServer() {
         throwError("m_szWLDisplaySocket was null! (wl_display_add_socket and wl_display_add_socket_auto failed)");
     }
 
-    Debug::log(LOG, "Setting WAYLAND_DISPLAY to {}", m_szWLDisplaySocket);
     setenv("WAYLAND_DISPLAY", m_szWLDisplaySocket.c_str(), 1);
     setenv("XDG_SESSION_TYPE", "wayland", 1);
 
@@ -631,44 +639,7 @@ void CCompositor::prepareFallbackOutput() {
     headless->createOutput();
 }
 
-void CCompositor::startCompositor(std::string socketName, int socketFd) {
-    if (!socketName.empty() && socketFd != -1) {
-        fcntl(socketFd, F_SETFD, FD_CLOEXEC);
-        const auto RETVAL = wl_display_add_socket_fd(m_sWLDisplay, socketFd);
-        if (RETVAL >= 0) {
-            m_szWLDisplaySocket = socketName;
-            Debug::log(LOG, "wl_display_add_socket_fd for {} succeeded with {}", socketName, RETVAL);
-        } else
-            Debug::log(WARN, "wl_display_add_socket_fd for {} returned {}: skipping", socketName, RETVAL);
-    } else {
-        // get socket, avoid using 0
-        for (int candidate = 1; candidate <= 32; candidate++) {
-            const auto CANDIDATESTR = ("wayland-" + std::to_string(candidate));
-            const auto RETVAL       = wl_display_add_socket(m_sWLDisplay, CANDIDATESTR.c_str());
-            if (RETVAL >= 0) {
-                m_szWLDisplaySocket = CANDIDATESTR;
-                Debug::log(LOG, "wl_display_add_socket for {} succeeded with {}", CANDIDATESTR, RETVAL);
-                break;
-            } else
-                Debug::log(WARN, "wl_display_add_socket for {} returned {}: skipping candidate {}", CANDIDATESTR, RETVAL, candidate);
-        }
-    }
-
-    if (m_szWLDisplaySocket.empty()) {
-        Debug::log(WARN, "All candidates failed, trying wl_display_add_socket_auto");
-        const auto SOCKETSTR = wl_display_add_socket_auto(m_sWLDisplay);
-        if (SOCKETSTR)
-            m_szWLDisplaySocket = SOCKETSTR;
-    }
-
-    if (m_szWLDisplaySocket.empty()) {
-        Debug::log(CRIT, "m_szWLDisplaySocket NULL!");
-        throwError("m_szWLDisplaySocket was null! (wl_display_add_socket and wl_display_add_socket_auto failed)");
-    }
-
-    setenv("WAYLAND_DISPLAY", m_szWLDisplaySocket.c_str(), 1);
-    setenv("XDG_SESSION_TYPE", "wayland", 1);
-
+void CCompositor::startCompositor() {
     signal(SIGPIPE, SIG_IGN);
 
     if (m_pAqBackend->hasSession() /* Session-less Hyprland usually means a nest, don't update the env in that case */) {
diff --git a/src/Compositor.hpp b/src/Compositor.hpp
index 58c5b33d..a274ca58 100644
--- a/src/Compositor.hpp
+++ b/src/Compositor.hpp
@@ -69,8 +69,8 @@ class CCompositor {
 
     std::unordered_map<std::string, uint64_t> m_mMonitorIDMap;
 
-    void                                      initServer();
-    void                                      startCompositor(std::string socketName, int socketFd);
+    void                                      initServer(std::string socketName, int socketFd);
+    void                                      startCompositor();
     void                                      cleanup();
     void                                      createLockFile();
     void                                      removeLockFile();
diff --git a/src/main.cpp b/src/main.cpp
index 1ac3ab8b..3e51c6c8 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -147,7 +147,7 @@ int main(int argc, char** argv) {
         return 1;
     }
 
-    g_pCompositor->initServer();
+    g_pCompositor->initServer(socketName, socketFd);
 
     if (!envEnabled("HYPRLAND_NO_RT"))
         Init::gainRealTime();
@@ -155,7 +155,7 @@ int main(int argc, char** argv) {
     Debug::log(LOG, "Hyprland init finished.");
 
     // If all's good to go, start.
-    g_pCompositor->startCompositor(socketName, socketFd);
+    g_pCompositor->startCompositor();
 
     g_pCompositor->m_bIsShuttingDown = true;