shadows: fix on transformed

4 files changed, 100 insertions, 31 deletions

diff --git a/src/render/OpenGL.cpp b/src/render/OpenGL.cpp
index 0011e636..8c691b35 100644
--- a/src/render/OpenGL.cpp
+++ b/src/render/OpenGL.cpp
@@ -235,8 +235,10 @@ void CHyprOpenGLImpl::initShaders() {
     m_RenderData.pCurrentMonData->m_shRGBA.program           = prog;
     m_RenderData.pCurrentMonData->m_shRGBA.proj              = glGetUniformLocation(prog, "proj");
     m_RenderData.pCurrentMonData->m_shRGBA.tex               = glGetUniformLocation(prog, "tex");
+    m_RenderData.pCurrentMonData->m_shRGBA.alphaMatte        = glGetUniformLocation(prog, "texMatte");
     m_RenderData.pCurrentMonData->m_shRGBA.alpha             = glGetUniformLocation(prog, "alpha");
     m_RenderData.pCurrentMonData->m_shRGBA.texAttrib         = glGetAttribLocation(prog, "texcoord");
+    m_RenderData.pCurrentMonData->m_shRGBA.matteTexAttrib    = glGetAttribLocation(prog, "texcoordMatte");
     m_RenderData.pCurrentMonData->m_shRGBA.posAttrib         = glGetAttribLocation(prog, "pos");
     m_RenderData.pCurrentMonData->m_shRGBA.discardOpaque     = glGetUniformLocation(prog, "discardOpaque");
     m_RenderData.pCurrentMonData->m_shRGBA.discardAlpha      = glGetUniformLocation(prog, "discardAlpha");
@@ -246,6 +248,7 @@ void CHyprOpenGLImpl::initShaders() {
     m_RenderData.pCurrentMonData->m_shRGBA.radius            = glGetUniformLocation(prog, "radius");
     m_RenderData.pCurrentMonData->m_shRGBA.applyTint         = glGetUniformLocation(prog, "applyTint");
     m_RenderData.pCurrentMonData->m_shRGBA.tint              = glGetUniformLocation(prog, "tint");
+    m_RenderData.pCurrentMonData->m_shRGBA.useAlphaMatte     = glGetUniformLocation(prog, "useAlphaMatte");
 
     prog                                                     = createProgram(TEXVERTSRC, TEXFRAGSRCRGBAPASSTHRU);
     m_RenderData.pCurrentMonData->m_shPASSTHRURGBA.program   = prog;
@@ -254,6 +257,14 @@ void CHyprOpenGLImpl::initShaders() {
     m_RenderData.pCurrentMonData->m_shPASSTHRURGBA.texAttrib = glGetAttribLocation(prog, "texcoord");
     m_RenderData.pCurrentMonData->m_shPASSTHRURGBA.posAttrib = glGetAttribLocation(prog, "pos");
 
+    prog                                               = createProgram(TEXVERTSRC, TEXFRAGSRCRGBAMATTE);
+    m_RenderData.pCurrentMonData->m_shMATTE.program    = prog;
+    m_RenderData.pCurrentMonData->m_shMATTE.proj       = glGetUniformLocation(prog, "proj");
+    m_RenderData.pCurrentMonData->m_shMATTE.tex        = glGetUniformLocation(prog, "tex");
+    m_RenderData.pCurrentMonData->m_shMATTE.alphaMatte = glGetUniformLocation(prog, "texMatte");
+    m_RenderData.pCurrentMonData->m_shMATTE.texAttrib  = glGetAttribLocation(prog, "texcoord");
+    m_RenderData.pCurrentMonData->m_shMATTE.posAttrib  = glGetAttribLocation(prog, "pos");
+
     prog                                               = createProgram(TEXVERTSRC, FRAGGLITCH);
     m_RenderData.pCurrentMonData->m_shGLITCH.program   = prog;
     m_RenderData.pCurrentMonData->m_shGLITCH.proj      = glGetUniformLocation(prog, "proj");
@@ -821,6 +832,64 @@ void CHyprOpenGLImpl::renderTexturePrimitive(const CTexture& tex, CBox* pBox) {
     glBindTexture(tex.m_iTarget, 0);
 }
 
+void CHyprOpenGLImpl::renderTextureMatte(const CTexture& tex, CBox* pBox, CFramebuffer& matte) {
+    RASSERT(m_RenderData.pMonitor, "Tried to render texture without begin()!");
+    RASSERT((tex.m_iTexID > 0), "Attempted to draw NULL texture!");
+
+    TRACY_GPU_ZONE("RenderTextureMatte");
+
+    if (m_RenderData.damage.empty())
+        return;
+
+    CBox newBox = *pBox;
+    newBox.scale(m_RenderData.renderModif.scale).translate(m_RenderData.renderModif.translate);
+
+    // get transform
+    const auto TRANSFORM = wlr_output_transform_invert(!m_bEndFrame ? WL_OUTPUT_TRANSFORM_NORMAL : m_RenderData.pMonitor->transform);
+    float      matrix[9];
+    wlr_matrix_project_box(matrix, newBox.pWlr(), TRANSFORM, 0, m_RenderData.pMonitor->output->transform_matrix);
+
+    float glMatrix[9];
+    wlr_matrix_multiply(glMatrix, m_RenderData.projection, matrix);
+
+    CShader* shader = &m_RenderData.pCurrentMonData->m_shMATTE;
+
+    glUseProgram(shader->program);
+
+#ifndef GLES2
+    glUniformMatrix3fv(shader->proj, 1, GL_TRUE, glMatrix);
+#else
+    wlr_matrix_transpose(glMatrix, glMatrix);
+    glUniformMatrix3fv(shader->proj, 1, GL_FALSE, glMatrix);
+#endif
+    glUniform1i(shader->tex, 0);
+    glUniform1i(shader->alphaMatte, 1);
+
+    glActiveTexture(GL_TEXTURE0);
+    glBindTexture(tex.m_iTarget, tex.m_iTexID);
+
+    glActiveTexture(GL_TEXTURE0 + 1);
+    glBindTexture(matte.m_cTex.m_iTarget, matte.m_cTex.m_iTexID);
+
+    glVertexAttribPointer(shader->posAttrib, 2, GL_FLOAT, GL_FALSE, 0, fullVerts);
+    glVertexAttribPointer(shader->texAttrib, 2, GL_FLOAT, GL_FALSE, 0, fullVerts);
+
+    glEnableVertexAttribArray(shader->posAttrib);
+    glEnableVertexAttribArray(shader->texAttrib);
+
+    for (auto& RECT : m_RenderData.damage.getRects()) {
+        scissor(&RECT);
+        glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
+    }
+
+    scissor((CBox*)nullptr);
+
+    glDisableVertexAttribArray(shader->posAttrib);
+    glDisableVertexAttribArray(shader->texAttrib);
+
+    glBindTexture(tex.m_iTarget, 0);
+}
+
 // This probably isn't the fastest
 // but it works... well, I guess?
 //
@@ -1629,7 +1698,7 @@ void CHyprOpenGLImpl::renderSnapshot(SLayerSurface** pLayer) {
     m_bEndFrame = false;
 }
 
-void CHyprOpenGLImpl::renderRoundedShadow(CBox* box, int round, int range, float a, CFramebuffer* matte) {
+void CHyprOpenGLImpl::renderRoundedShadow(CBox* box, int round, int range, float a) {
     RASSERT(m_RenderData.pMonitor, "Tried to render shadow without begin()!");
     RASSERT((box->width > 0 && box->height > 0), "Tried to render shadow with width/height < 0!");
     RASSERT(m_pCurrentWindow, "Tried to render shadow without a window!");
@@ -1647,7 +1716,6 @@ void CHyprOpenGLImpl::renderRoundedShadow(CBox* box, int round, int range, float
     static auto* const PSHADOWPOWER = &g_pConfigManager->getConfigValuePtr("decoration:shadow_render_power")->intValue;
 
     const auto         SHADOWPOWER = std::clamp((int)*PSHADOWPOWER, 1, 4);
-    const auto         USEMATTE    = matte;
 
     const auto         col = m_pCurrentWindow->m_cRealShadowColor.col();
 
@@ -1682,33 +1750,11 @@ void CHyprOpenGLImpl::renderRoundedShadow(CBox* box, int round, int range, float
     glUniform1f(m_RenderData.pCurrentMonData->m_shSHADOW.range, range);
     glUniform1f(m_RenderData.pCurrentMonData->m_shSHADOW.shadowPower, SHADOWPOWER);
 
-    if (USEMATTE) {
-        glUniform1i(m_RenderData.pCurrentMonData->m_shSHADOW.useAlphaMatte, 1);
-        glUniform1i(m_RenderData.pCurrentMonData->m_shSHADOW.alphaMatte, 0);
-        glActiveTexture(GL_TEXTURE0);
-        glBindTexture(matte->m_cTex.m_iTarget, matte->m_cTex.m_iTexID);
-    } else {
-        glUniform1i(m_RenderData.pCurrentMonData->m_shSHADOW.useAlphaMatte, 0);
-    }
-
-    const float texVerts[] = {
-        ((float)(box->x + box->width) / m_RenderData.pMonitor->vecPixelSize.x),
-        ((float)box->y / m_RenderData.pMonitor->vecPixelSize.y), // top right
-        ((float)box->x / m_RenderData.pMonitor->vecPixelSize.x),
-        ((float)box->y / m_RenderData.pMonitor->vecPixelSize.y), // top left
-        ((float)(box->x + box->width) / m_RenderData.pMonitor->vecPixelSize.x),
-        ((float)(box->y + box->height) / m_RenderData.pMonitor->vecPixelSize.y), // bottom right
-        ((float)box->x / m_RenderData.pMonitor->vecPixelSize.x),
-        ((float)(box->y + box->height) / m_RenderData.pMonitor->vecPixelSize.y), // bottom left
-    };
-
     glVertexAttribPointer(m_RenderData.pCurrentMonData->m_shSHADOW.posAttrib, 2, GL_FLOAT, GL_FALSE, 0, fullVerts);
     glVertexAttribPointer(m_RenderData.pCurrentMonData->m_shSHADOW.texAttrib, 2, GL_FLOAT, GL_FALSE, 0, fullVerts);
-    glVertexAttribPointer(m_RenderData.pCurrentMonData->m_shSHADOW.matteTexAttrib, 2, GL_FLOAT, GL_FALSE, 0, texVerts);
 
     glEnableVertexAttribArray(m_RenderData.pCurrentMonData->m_shSHADOW.posAttrib);
     glEnableVertexAttribArray(m_RenderData.pCurrentMonData->m_shSHADOW.texAttrib);
-    glEnableVertexAttribArray(m_RenderData.pCurrentMonData->m_shSHADOW.matteTexAttrib);
 
     if (m_RenderData.clipBox.width != 0 && m_RenderData.clipBox.height != 0) {
         CRegion damageClip{m_RenderData.clipBox.x, m_RenderData.clipBox.y, m_RenderData.clipBox.width, m_RenderData.clipBox.height};
@@ -1727,7 +1773,6 @@ void CHyprOpenGLImpl::renderRoundedShadow(CBox* box, int round, int range, float
         }
     }
 
-    glDisableVertexAttribArray(m_RenderData.pCurrentMonData->m_shSHADOW.matteTexAttrib);
     glDisableVertexAttribArray(m_RenderData.pCurrentMonData->m_shSHADOW.posAttrib);
     glDisableVertexAttribArray(m_RenderData.pCurrentMonData->m_shSHADOW.texAttrib);
 }
diff --git a/src/render/OpenGL.hpp b/src/render/OpenGL.hpp
index 93dccbf4..fea967e5 100644
--- a/src/render/OpenGL.hpp
+++ b/src/render/OpenGL.hpp
@@ -59,6 +59,7 @@ struct SMonitorRenderData {
     CShader m_shQUAD;
     CShader m_shRGBA;
     CShader m_shPASSTHRURGBA;
+    CShader m_shMATTE;
     CShader m_shRGBX;
     CShader m_shEXT;
     CShader m_shBLUR1;
@@ -96,6 +97,7 @@ struct SCurrentRenderData {
 };
 
 class CGradientValueData;
+class CHyprDropShadowDecoration;
 
 class CHyprOpenGLImpl {
   public:
@@ -111,7 +113,7 @@ class CHyprOpenGLImpl {
     void               renderTexture(wlr_texture*, CBox*, float a, int round = 0, bool allowCustomUV = false);
     void               renderTexture(const CTexture&, CBox*, float a, int round = 0, bool discardActive = false, bool allowCustomUV = false);
     void               renderTextureWithBlur(const CTexture&, CBox*, float a, wlr_surface* pSurface, int round = 0, bool blockBlurOptimization = false, float blurA = 1.f);
-    void               renderRoundedShadow(CBox*, int round, int range, float a = 1.0, CFramebuffer* matte = nullptr);
+    void               renderRoundedShadow(CBox*, int round, int range, float a = 1.0);
     void               renderBorder(CBox*, const CGradientValueData&, int round, int borderSize, float a = 1.0, int outerRound = -1 /* use round */);
 
     void               saveMatrix();
@@ -190,6 +192,7 @@ class CHyprOpenGLImpl {
     void          renderTextureInternalWithDamage(const CTexture&, CBox* pBox, float a, CRegion* damage, int round = 0, bool discardOpaque = false, bool noAA = false,
                                                   bool allowCustomUV = false, bool allowDim = false);
     void          renderTexturePrimitive(const CTexture& tex, CBox* pBox);
+    void          renderTextureMatte(const CTexture& tex, CBox* pBox, CFramebuffer& matte);
     void          renderSplash(cairo_t* const, cairo_surface_t* const, double);
 
     void          preBlurForCurrentMonitor();
@@ -197,6 +200,7 @@ class CHyprOpenGLImpl {
     bool          shouldUseNewBlurOptimizations(SLayerSurface* pLayer, CWindow* pWindow);
 
     friend class CHyprRenderer;
+    friend class CHyprDropShadowDecoration;
 };
 
 inline std::unique_ptr<CHyprOpenGLImpl> g_pHyprOpenGL;
diff --git a/src/render/decorations/CHyprDropShadowDecoration.cpp b/src/render/decorations/CHyprDropShadowDecoration.cpp
index 38feb512..74f14e0c 100644
--- a/src/render/decorations/CHyprDropShadowDecoration.cpp
+++ b/src/render/decorations/CHyprDropShadowDecoration.cpp
@@ -126,8 +126,9 @@ void CHyprDropShadowDecoration::draw(CMonitor* pMonitor, float a, const Vector2D
     g_pHyprOpenGL->scissor((CBox*)nullptr);
 
     // we'll take the liberty of using this as it should not be used rn
-    CFramebuffer& alphaFB = g_pHyprOpenGL->m_RenderData.pCurrentMonData->mirrorFB;
-    auto*         LASTFB  = g_pHyprOpenGL->m_RenderData.currentFB;
+    CFramebuffer& alphaFB     = g_pHyprOpenGL->m_RenderData.pCurrentMonData->mirrorFB;
+    CFramebuffer& alphaSwapFB = g_pHyprOpenGL->m_RenderData.pCurrentMonData->mirrorSwapFB;
+    auto*         LASTFB      = g_pHyprOpenGL->m_RenderData.currentFB;
 
     if (*PSHADOWIGNOREWINDOW) {
         CBox windowBox = m_bLastWindowBox;
@@ -146,11 +147,20 @@ void CHyprDropShadowDecoration::draw(CMonitor* pMonitor, float a, const Vector2D
 
         g_pHyprOpenGL->renderRect(&windowBox, CColor(1.0, 1.0, 1.0, 1.0), ROUNDING * pMonitor->scale);
 
+        alphaSwapFB.bind();
+        g_pHyprOpenGL->clear(CColor(0, 0, 0, 0));
+
+        g_pHyprOpenGL->renderRoundedShadow(&fullBox, ROUNDING * pMonitor->scale, *PSHADOWSIZE * pMonitor->scale, a);
+
         LASTFB->bind();
-    }
 
-    fullBox.scale(pMonitor->scale).round();
-    g_pHyprOpenGL->renderRoundedShadow(&fullBox, ROUNDING * pMonitor->scale, *PSHADOWSIZE * pMonitor->scale, a, *PSHADOWIGNOREWINDOW ? &alphaFB : nullptr);
+        CBox monbox                = {0, 0, pMonitor->vecTransformedSize.x, pMonitor->vecTransformedSize.y};
+        g_pHyprOpenGL->m_bEndFrame = true;
+        g_pHyprOpenGL->renderTextureMatte(alphaSwapFB.m_cTex, &monbox, alphaFB);
+        g_pHyprOpenGL->m_bEndFrame = false;
+    } else {
+        g_pHyprOpenGL->renderRoundedShadow(&fullBox, ROUNDING * pMonitor->scale, *PSHADOWSIZE * pMonitor->scale, a);
+    }
 }
 
 eDecorationLayer CHyprDropShadowDecoration::getDecorationLayer() {
diff --git a/src/render/shaders/Textures.hpp b/src/render/shaders/Textures.hpp
index 6f19ac9f..cf753bb4 100644
--- a/src/render/shaders/Textures.hpp
+++ b/src/render/shaders/Textures.hpp
@@ -130,6 +130,16 @@ void main() {
     gl_FragColor = texture2D(tex, v_texcoord);
 })#";
 
+inline const std::string TEXFRAGSRCRGBAMATTE = R"#(
+precision mediump float;
+varying vec2 v_texcoord; // is in 0-1
+uniform sampler2D tex;
+uniform sampler2D texMatte;
+
+void main() {
+    gl_FragColor = texture2D(tex, v_texcoord) * (1.0 - texture2D(texMatte, v_texcoord)[3]);
+})#";
+
 inline const std::string TEXFRAGSRCRGBX = R"#(
 precision mediump float;
 varying vec2 v_texcoord;