renderer: properly pass 10-bit formats to opengl

7 files changed, 33 insertions, 13 deletions

diff --git a/src/helpers/MiscFunctions.cpp b/src/helpers/MiscFunctions.cpp
index 3cce73d2..acb9cf20 100644
--- a/src/helpers/MiscFunctions.cpp
+++ b/src/helpers/MiscFunctions.cpp
@@ -750,4 +750,16 @@ std::vector<SCallstackFrameInfo> getBacktrace() {
 void throwError(const std::string& err) {
     Debug::log(CRIT, "Critical error thrown: {}", err);
     throw std::runtime_error(err);
+}
+
+uint32_t drmFormatToGL(uint32_t drm) {
+    switch (drm) {
+        case DRM_FORMAT_XRGB8888:
+        case DRM_FORMAT_XBGR8888: return GL_RGBA; // doesn't matter, opengl is gucci in this case.
+        case DRM_FORMAT_XRGB2101010:
+        case DRM_FORMAT_XBGR2101010: return GL_RGB10_A2;
+        default: return GL_RGBA;
+    }
+    UNREACHABLE();
+    return GL_RGBA;
 }
\ No newline at end of file
diff --git a/src/helpers/MiscFunctions.hpp b/src/helpers/MiscFunctions.hpp
index d489a0cd..499aa90f 100644
--- a/src/helpers/MiscFunctions.hpp
+++ b/src/helpers/MiscFunctions.hpp
@@ -33,6 +33,7 @@ double                           normalizeAngleRad(double ang);
 std::string                      replaceInString(std::string subject, const std::string& search, const std::string& replace);
 std::vector<SCallstackFrameInfo> getBacktrace();
 void                             throwError(const std::string& err);
+uint32_t                         drmFormatToGL(uint32_t drm);
 
 template <typename... Args>
 [[deprecated("use std::format instead")]] std::string getFormat(std::format_string<Args...> fmt, Args&&... args) {
diff --git a/src/helpers/Monitor.hpp b/src/helpers/Monitor.hpp
index 71c84273..5e0df858 100644
--- a/src/helpers/Monitor.hpp
+++ b/src/helpers/Monitor.hpp
@@ -65,6 +65,7 @@ class CMonitor {
     bool                vrrActive     = false; // this can be TRUE even if VRR is not active in the case that this display does not support it.
     bool                enabled10bit  = false; // as above, this can be TRUE even if 10 bit failed.
     bool                createdByUser = false;
+    uint32_t            drmFormat     = DRM_FORMAT_INVALID;
 
     bool                pendingFrame    = false; // if we schedule a frame during rendering, reschedule it after
     bool                renderingActive = false;
diff --git a/src/render/Framebuffer.cpp b/src/render/Framebuffer.cpp
index a62ea18a..e1f815ce 100644
--- a/src/render/Framebuffer.cpp
+++ b/src/render/Framebuffer.cpp
@@ -1,10 +1,13 @@
 #include "Framebuffer.hpp"
 #include "OpenGL.hpp"
 
-bool CFramebuffer::alloc(int w, int h) {
+bool CFramebuffer::alloc(int w, int h, uint32_t drmFormat) {
     bool firstAlloc = false;
     RASSERT((w > 1 && h > 1), "cannot alloc a FB with negative / zero size! (attempted {}x{})", w, h);
 
+    uint32_t glFormat = drmFormatToGL(drmFormat);
+    uint32_t glType   = glFormat != GL_RGBA ? GL_UNSIGNED_INT_2_10_10_10_REV : GL_UNSIGNED_BYTE;
+
     if (m_iFb == (uint32_t)-1) {
         firstAlloc = true;
         glGenFramebuffers(1, &m_iFb);
@@ -22,7 +25,7 @@ bool CFramebuffer::alloc(int w, int h) {
 
     if (firstAlloc || m_vSize != Vector2D(w, h)) {
         glBindTexture(GL_TEXTURE_2D, m_cTex.m_iTexID);
-        glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, w, h, 0, GL_RGBA, GL_UNSIGNED_BYTE, 0);
+        glTexImage2D(GL_TEXTURE_2D, 0, glFormat, w, h, 0, GL_RGBA, glType, 0);
 
         glBindFramebuffer(GL_FRAMEBUFFER, m_iFb);
         glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, m_cTex.m_iTexID, 0);
diff --git a/src/render/Framebuffer.hpp b/src/render/Framebuffer.hpp
index 9ba3ac05..73b70431 100644
--- a/src/render/Framebuffer.hpp
+++ b/src/render/Framebuffer.hpp
@@ -7,7 +7,7 @@ class CFramebuffer {
   public:
     ~CFramebuffer();
 
-    bool      alloc(int w, int h);
+    bool      alloc(int w, int h, uint32_t format = GL_RGBA);
     void      bind();
     void      release();
     void      reset();
diff --git a/src/render/OpenGL.cpp b/src/render/OpenGL.cpp
index 4a5b70dd..d23a1d38 100644
--- a/src/render/OpenGL.cpp
+++ b/src/render/OpenGL.cpp
@@ -130,10 +130,10 @@ void CHyprOpenGLImpl::begin(CMonitor* pMonitor, CRegion* pDamage, bool fake) {
         m_RenderData.pCurrentMonData->mirrorSwapFB.m_pStencilTex = &m_RenderData.pCurrentMonData->stencilTex;
         m_RenderData.pCurrentMonData->offMainFB.m_pStencilTex    = &m_RenderData.pCurrentMonData->stencilTex;
 
-        m_RenderData.pCurrentMonData->primaryFB.alloc(pMonitor->vecPixelSize.x, pMonitor->vecPixelSize.y);
-        m_RenderData.pCurrentMonData->mirrorFB.alloc(pMonitor->vecPixelSize.x, pMonitor->vecPixelSize.y);
-        m_RenderData.pCurrentMonData->mirrorSwapFB.alloc(pMonitor->vecPixelSize.x, pMonitor->vecPixelSize.y);
-        m_RenderData.pCurrentMonData->offMainFB.alloc(pMonitor->vecPixelSize.x, pMonitor->vecPixelSize.y);
+        m_RenderData.pCurrentMonData->primaryFB.alloc(pMonitor->vecPixelSize.x, pMonitor->vecPixelSize.y, pMonitor->drmFormat);
+        m_RenderData.pCurrentMonData->mirrorFB.alloc(pMonitor->vecPixelSize.x, pMonitor->vecPixelSize.y, pMonitor->drmFormat);
+        m_RenderData.pCurrentMonData->mirrorSwapFB.alloc(pMonitor->vecPixelSize.x, pMonitor->vecPixelSize.y, pMonitor->drmFormat);
+        m_RenderData.pCurrentMonData->offMainFB.alloc(pMonitor->vecPixelSize.x, pMonitor->vecPixelSize.y, pMonitor->drmFormat);
 
         createBGTextureForMonitor(pMonitor);
     }
@@ -1139,7 +1139,7 @@ void CHyprOpenGLImpl::preBlurForCurrentMonitor() {
     const auto POUTFB       = blurMainFramebufferWithDamage(1, &fakeDamage);
 
     // render onto blurFB
-    m_RenderData.pCurrentMonData->blurFB.alloc(m_RenderData.pMonitor->vecPixelSize.x, m_RenderData.pMonitor->vecPixelSize.y);
+    m_RenderData.pCurrentMonData->blurFB.alloc(m_RenderData.pMonitor->vecPixelSize.x, m_RenderData.pMonitor->vecPixelSize.y, m_RenderData.pMonitor->drmFormat);
     m_RenderData.pCurrentMonData->blurFB.bind();
 
     clear(CColor(0, 0, 0, 0));
@@ -1430,7 +1430,7 @@ void CHyprOpenGLImpl::makeRawWindowSnapshot(CWindow* pWindow, CFramebuffer* pFra
 
     pFramebuffer->m_pStencilTex = &m_RenderData.pCurrentMonData->stencilTex;
 
-    pFramebuffer->alloc(PMONITOR->vecPixelSize.x, PMONITOR->vecPixelSize.y);
+    pFramebuffer->alloc(PMONITOR->vecPixelSize.x, PMONITOR->vecPixelSize.y, PMONITOR->drmFormat);
 
     pFramebuffer->bind();
 
@@ -1488,7 +1488,7 @@ void CHyprOpenGLImpl::makeWindowSnapshot(CWindow* pWindow) {
 
     PFRAMEBUFFER->m_pStencilTex = &m_RenderData.pCurrentMonData->stencilTex;
 
-    PFRAMEBUFFER->alloc(PMONITOR->vecPixelSize.x, PMONITOR->vecPixelSize.y);
+    PFRAMEBUFFER->alloc(PMONITOR->vecPixelSize.x, PMONITOR->vecPixelSize.y, PMONITOR->drmFormat);
 
     PFRAMEBUFFER->bind();
 
@@ -1533,7 +1533,7 @@ void CHyprOpenGLImpl::makeLayerSnapshot(SLayerSurface* pLayer) {
 
     glViewport(0, 0, g_pHyprOpenGL->m_RenderData.pMonitor->vecPixelSize.x, g_pHyprOpenGL->m_RenderData.pMonitor->vecPixelSize.y);
 
-    PFRAMEBUFFER->alloc(PMONITOR->vecPixelSize.x, PMONITOR->vecPixelSize.y);
+    PFRAMEBUFFER->alloc(PMONITOR->vecPixelSize.x, PMONITOR->vecPixelSize.y, PMONITOR->drmFormat);
 
     PFRAMEBUFFER->bind();
 
@@ -1721,7 +1721,7 @@ void CHyprOpenGLImpl::renderRoundedShadow(wlr_box* box, int round, int range, fl
 void CHyprOpenGLImpl::saveBufferForMirror() {
 
     if (!m_RenderData.pCurrentMonData->monitorMirrorFB.isAllocated())
-        m_RenderData.pCurrentMonData->monitorMirrorFB.alloc(m_RenderData.pMonitor->vecPixelSize.x, m_RenderData.pMonitor->vecPixelSize.y);
+        m_RenderData.pCurrentMonData->monitorMirrorFB.alloc(m_RenderData.pMonitor->vecPixelSize.x, m_RenderData.pMonitor->vecPixelSize.y, m_RenderData.pMonitor->drmFormat);
 
     m_RenderData.pCurrentMonData->monitorMirrorFB.bind();
 
diff --git a/src/render/Renderer.cpp b/src/render/Renderer.cpp
index 927c589d..912c1bed 100644
--- a/src/render/Renderer.cpp
+++ b/src/render/Renderer.cpp
@@ -1877,7 +1877,8 @@ bool CHyprRenderer::applyMonitorRule(CMonitor* pMonitor, SMonitorRule* pMonitorR
     };
     // clang-format on
 
-    bool set10bit = false;
+    bool set10bit       = false;
+    pMonitor->drmFormat = DRM_FORMAT_INVALID;
 
     for (auto& fmt : formats[(int)!pMonitorRule->enable10bit]) {
         wlr_output_set_render_format(pMonitor->output, fmt.second);
@@ -1888,6 +1889,8 @@ bool CHyprRenderer::applyMonitorRule(CMonitor* pMonitor, SMonitorRule* pMonitorR
             Debug::log(LOG, "output {} succeeded basic test on format {}", pMonitor->szName, fmt.first);
             if (pMonitorRule->enable10bit && fmt.first.contains("101010"))
                 set10bit = true;
+
+            pMonitor->drmFormat = fmt.second;
             break;
         }
     }