pointer: add drm dumb buffers for cursors (#8399)

---------

Co-authored-by: Mihai Fufezan <[email protected]>

1 files changed, 76 insertions, 50 deletions

diff --git a/src/managers/PointerManager.cpp b/src/managers/PointerManager.cpp
index 75be235f..0580c9be 100644
--- a/src/managers/PointerManager.cpp
+++ b/src/managers/PointerManager.cpp
@@ -375,6 +375,8 @@ SP<Aquamarine::IBuffer> CPointerManager::renderHWCursorBuffer(SP<CPointerManager
     auto        maxSize    = state->monitor->output->cursorPlaneSize();
     auto const& cursorSize = currentCursorImage.size;
 
+    static auto PDUMB = CConfigValue<Hyprlang::INT>("cursor:use_cpu_buffer");
+
     if (maxSize == Vector2D{})
         return nullptr;
 
@@ -386,10 +388,23 @@ SP<Aquamarine::IBuffer> CPointerManager::renderHWCursorBuffer(SP<CPointerManager
     } else
         maxSize = cursorSize;
 
-    if (!state->monitor->cursorSwapchain || maxSize != state->monitor->cursorSwapchain->currentOptions().size) {
+    if (!state->monitor->cursorSwapchain || maxSize != state->monitor->cursorSwapchain->currentOptions().size ||
+        *PDUMB != (state->monitor->cursorSwapchain->getAllocator()->type() != Aquamarine::AQ_ALLOCATOR_TYPE_GBM)) {
+
+        if (!state->monitor->cursorSwapchain || *PDUMB != (state->monitor->cursorSwapchain->getAllocator()->type() != Aquamarine::AQ_ALLOCATOR_TYPE_GBM)) {
+
+            auto allocator = state->monitor->output->getBackend()->preferredAllocator();
+            if (*PDUMB) {
+                for (const auto& a : state->monitor->output->getBackend()->getAllocators()) {
+                    if (a->type() == Aquamarine::AQ_ALLOCATOR_TYPE_DRM_DUMB) {
+                        allocator = a;
+                        break;
+                    }
+                }
+            }
 
-        if (!state->monitor->cursorSwapchain)
-            state->monitor->cursorSwapchain = Aquamarine::CSwapchain::create(state->monitor->output->getBackend()->preferredAllocator(), state->monitor->output->getBackend());
+            state->monitor->cursorSwapchain = Aquamarine::CSwapchain::create(allocator, state->monitor->output->getBackend());
+        }
 
         auto options     = state->monitor->cursorSwapchain->currentOptions();
         options.size     = maxSize;
@@ -397,8 +412,10 @@ SP<Aquamarine::IBuffer> CPointerManager::renderHWCursorBuffer(SP<CPointerManager
         options.scanout  = true;
         options.cursor   = true;
         options.multigpu = state->monitor->output->getBackend()->preferredAllocator()->drmFD() != g_pCompositor->m_iDRMFD;
-        // We do not set the format. If it's unset (DRM_FORMAT_INVALID) then the swapchain will pick for us,
+        // We do not set the format (unless shm). If it's unset (DRM_FORMAT_INVALID) then the swapchain will pick for us,
         // but if it's set, we don't wanna change it.
+        if (*PDUMB)
+            options.format = DRM_FORMAT_ARGB8888;
 
         if (!state->monitor->cursorSwapchain->reconfigure(options)) {
             Debug::log(TRACE, "Failed to reconfigure cursor swapchain");
@@ -420,62 +437,71 @@ SP<Aquamarine::IBuffer> CPointerManager::renderHWCursorBuffer(SP<CPointerManager
         return nullptr;
     }
 
-    g_pHyprRenderer->makeEGLCurrent();
-    g_pHyprOpenGL->m_RenderData.pMonitor = state->monitor;
-
-    auto RBO = g_pHyprRenderer->getOrCreateRenderbuffer(buf, state->monitor->cursorSwapchain->currentOptions().format);
-    if (!RBO) {
-        Debug::log(TRACE, "Failed to create cursor RB with format {}, mod {}", buf->dmabuf().format, buf->dmabuf().modifier);
-        static auto PDUMB = CConfigValue<Hyprlang::INT>("cursor:allow_dumb_copy");
-        if (!*PDUMB)
-            return nullptr;
-
-        auto bufData = buf->beginDataPtr(0);
-        auto bufPtr  = std::get<0>(bufData);
-
-        // clear buffer
-        memset(bufPtr, 0, std::get<2>(bufData));
-
-        if (currentCursorImage.pBuffer) {
-            auto texAttrs = currentCursorImage.pBuffer->shm();
-
-            if (!texAttrs.success) {
+    if (*PDUMB) {
+        // get the texture data if available.
+        auto texData = texture->dataCopy();
+        if (texData.empty()) {
+            if (currentCursorImage.surface && currentCursorImage.surface->resource()->role->role() == SURFACE_ROLE_CURSOR) {
+                const auto SURFACE   = currentCursorImage.surface->resource();
+                auto&      shmBuffer = CCursorSurfaceRole::cursorPixelData(SURFACE);
+
+                bool       flipRB = false;
+
+                if (SURFACE->current.texture) {
+                    Debug::log(TRACE, "Cursor CPU surface: format {}, expecting AR24", FormatUtils::drmFormatName(SURFACE->current.texture->m_iDrmFormat));
+                    if (SURFACE->current.texture->m_iDrmFormat == DRM_FORMAT_ABGR8888) {
+                        Debug::log(TRACE, "Cursor CPU surface format AB24, will flip. WARNING: this will break on big endian!");
+                        flipRB = true;
+                    } else if (SURFACE->current.texture->m_iDrmFormat != DRM_FORMAT_ARGB8888) {
+                        Debug::log(TRACE, "Cursor CPU surface format rejected, falling back to sw");
+                        return nullptr;
+                    }
+                }
+
+                if (shmBuffer.data())
+                    texData = shmBuffer;
+                else {
+                    texData.resize(texture->m_vSize.x * 4 * texture->m_vSize.y);
+                    memset(texData.data(), 0x00, texData.size());
+                }
+
+                if (flipRB) {
+                    for (size_t i = 0; i < shmBuffer.size(); i += 4) {
+                        std::swap(shmBuffer.at(i), shmBuffer.at(i + 2)); // little-endian!!!!!!
+                    }
+                }
+            } else {
                 Debug::log(TRACE, "Cannot use dumb copy on dmabuf cursor buffers");
                 return nullptr;
             }
+        }
 
-            auto texData = currentCursorImage.pBuffer->beginDataPtr(GBM_BO_TRANSFER_WRITE);
-            auto texPtr  = std::get<0>(texData);
-            Debug::log(TRACE, "cursor texture {}x{} {} {} {}", texAttrs.size.x, texAttrs.size.y, (void*)texPtr, texAttrs.format, texAttrs.stride);
-            // copy cursor texture
-            for (int i = 0; i < texAttrs.size.y; i++)
-                memcpy(bufPtr + i * buf->dmabuf().strides[0], texPtr + i * texAttrs.stride, texAttrs.stride);
-        } else if (currentCursorImage.surface && currentCursorImage.surface->resource()->role->role() == SURFACE_ROLE_CURSOR) {
-            const auto SURFACE   = currentCursorImage.surface->resource();
-            auto&      shmBuffer = CCursorSurfaceRole::cursorPixelData(SURFACE);
-            Debug::log(TRACE, "cursor texture pixel data length: {}B", shmBuffer.size());
-
-            if (shmBuffer.data()) {
-                // copy cursor texture
-                // assume format is 32bpp
-                size_t STRIDE = 4 * SURFACE->current.bufferSize.x;
-                for (int i = 0; i < SURFACE->current.bufferSize.y; i++)
-                    memcpy(bufPtr + i * buf->dmabuf().strides[0], shmBuffer.data() + i * STRIDE, STRIDE);
-            } else {
-                // if there is no data, hide the cursor
-                memset(bufPtr, '\0', buf->size.x * buf->size.y * 4 /* assume 32bpp */);
-            }
+        // then, we just yeet it into the dumb buffer
 
-        } else {
-            Debug::log(TRACE, "Unsupported cursor buffer/surface, falling back to sw (can't dumb copy)");
-            return nullptr;
-        }
+        auto [data, fmt, size] = buf->beginDataPtr(0);
+
+        memset(data, 0, size);
+        if (buf->dmabuf().size.x > texture->m_vSize.x) {
+            size_t STRIDE = 4 * texture->m_vSize.x;
+            for (int i = 0; i < texture->m_vSize.y; i++)
+                memcpy(data + i * buf->dmabuf().strides[0], texData.data() + i * STRIDE, STRIDE);
+        } else
+            memcpy(data, texData.data(), std::min(size, texData.size()));
 
         buf->endDataPtr();
 
         return buf;
     }
 
+    g_pHyprRenderer->makeEGLCurrent();
+    g_pHyprOpenGL->m_RenderData.pMonitor = state->monitor;
+
+    auto RBO = g_pHyprRenderer->getOrCreateRenderbuffer(buf, state->monitor->cursorSwapchain->currentOptions().format);
+    if (!RBO) {
+        Debug::log(TRACE, "Failed to create cursor RB with format {}, mod {}", buf->dmabuf().format, buf->dmabuf().modifier);
+        return nullptr;
+    }
+
     RBO->bind();
 
     g_pHyprOpenGL->beginSimple(state->monitor.lock(), {0, 0, INT16_MAX, INT16_MAX}, RBO);
@@ -773,7 +799,7 @@ SP<CTexture> CPointerManager::getCurrentCursorTexture() {
 
     if (currentCursorImage.pBuffer) {
         if (!currentCursorImage.bufferTex)
-            currentCursorImage.bufferTex = makeShared<CTexture>(currentCursorImage.pBuffer);
+            currentCursorImage.bufferTex = makeShared<CTexture>(currentCursorImage.pBuffer, true);
         return currentCursorImage.bufferTex;
     }