rewritten borders, fixed msaa

5 files changed, 177 insertions, 90 deletions

diff --git a/src/render/OpenGL.cpp b/src/render/OpenGL.cpp
index e44e108b..a0e6483d 100644
--- a/src/render/OpenGL.cpp
+++ b/src/render/OpenGL.cpp
@@ -81,6 +81,12 @@ CHyprOpenGLImpl::CHyprOpenGLImpl() {
     m_shSHADOW.posAttrib = glGetAttribLocation(prog, "pos");
     m_shSHADOW.texAttrib = glGetAttribLocation(prog, "texcoord");
 
+    prog = createProgram(QUADVERTSRC, FRAGBORDER1);
+    m_shBORDER1.program = prog;
+    m_shBORDER1.proj = glGetUniformLocation(prog, "proj");
+    m_shBORDER1.posAttrib = glGetAttribLocation(prog, "pos");
+    m_shBORDER1.texAttrib = glGetAttribLocation(prog, "texcoord");
+
     Debug::log(LOG, "Shaders initialized successfully.");
 
     // End shaders
@@ -309,15 +315,15 @@ void CHyprOpenGLImpl::renderTexture(wlr_texture* tex, wlr_box* pBox, float alpha
     renderTexture(CTexture(tex), pBox, alpha, round);
 }
 
-void CHyprOpenGLImpl::renderTexture(const CTexture& tex, wlr_box* pBox, float alpha, int round, bool discardopaque, bool border, bool allowPrimary) {
+void CHyprOpenGLImpl::renderTexture(const CTexture& tex, wlr_box* pBox, float alpha, int round, bool discardopaque, bool allowPrimary) {
     RASSERT(m_RenderData.pMonitor, "Tried to render texture without begin()!");
 
-    renderTextureInternalWithDamage(tex, pBox, alpha, m_RenderData.pDamage, round, discardopaque, border, false, allowPrimary);
+    renderTextureInternalWithDamage(tex, pBox, alpha, m_RenderData.pDamage, round, discardopaque, false, allowPrimary);
 
     scissor((wlr_box*)nullptr);
 }
 
-void CHyprOpenGLImpl::renderTextureInternalWithDamage(const CTexture& tex, wlr_box* pBox, float alpha, pixman_region32_t* damage, int round, bool discardOpaque, bool border, bool noAA, bool allowPrimary) {
+void CHyprOpenGLImpl::renderTextureInternalWithDamage(const CTexture& tex, wlr_box* pBox, float alpha, pixman_region32_t* damage, int round, bool discardOpaque, bool noAA, bool allowPrimary) {
     RASSERT(m_RenderData.pMonitor, "Tried to render texture without begin()!");
     RASSERT((tex.m_iTexID > 0), "Attempted to draw NULL texture!");
 
@@ -351,23 +357,6 @@ void CHyprOpenGLImpl::renderTextureInternalWithDamage(const CTexture& tex, wlr_b
             RASSERT(false, "tex.m_iTarget unsupported!");
     }
 
-    // stencil for when we want a border
-    if (border) {
-        glClearStencil(0);
-        glClear(GL_STENCIL_BUFFER_BIT);
-
-        glEnable(GL_STENCIL_TEST);
-
-        glStencilFunc(GL_ALWAYS, 1, -1);
-        glStencilOp(GL_REPLACE, GL_REPLACE, GL_REPLACE);
-
-        // hacky fix to fix broken borders.
-        // TODO: this is kinda slow... question mark?
-        renderRect(pBox, CColor(0, 0, 0, 0), round);
-
-        glDisable(GL_STENCIL_TEST);
-    }
-
     glActiveTexture(GL_TEXTURE0);
     glBindTexture(tex.m_iTarget, tex.m_iTexID);
 
@@ -393,7 +382,7 @@ void CHyprOpenGLImpl::renderTextureInternalWithDamage(const CTexture& tex, wlr_b
     glUniform2f(glGetUniformLocation(shader->program, "bottomRight"), (float)BOTTOMRIGHT.x, (float)BOTTOMRIGHT.y);
     glUniform2f(glGetUniformLocation(shader->program, "fullSize"), (float)FULLSIZE.x, (float)FULLSIZE.y);
     glUniform1f(glGetUniformLocation(shader->program, "radius"), round);
-    glUniform1i(glGetUniformLocation(shader->program, "primitiveMultisample"), (int)(*PMULTISAMPLEEDGES == 1 && round != 0 && !border && !noAA));
+    glUniform1i(glGetUniformLocation(shader->program, "primitiveMultisample"), (int)(*PMULTISAMPLEEDGES == 1 && round != 0 && !noAA));
 
     glVertexAttribPointer(shader->posAttrib, 2, GL_FLOAT, GL_FALSE, 0, fullVerts);
 
@@ -421,30 +410,10 @@ void CHyprOpenGLImpl::renderTextureInternalWithDamage(const CTexture& tex, wlr_b
         }
     }
 
-    if (border) {
-        glEnable(GL_STENCIL_TEST);
-
-        glStencilFunc(GL_EQUAL, 1, -1);
-        glStencilOp(GL_KEEP, GL_KEEP, GL_REPLACE);
-    }
-
     glDisableVertexAttribArray(shader->posAttrib);
     glDisableVertexAttribArray(shader->texAttrib);
 
     glBindTexture(tex.m_iTarget, 0);
-
-    // if border draw
-    // we dont disable stencil here if we havent touched it. 
-    // some other func might be using it.
-    if (border) {
-        auto BORDERCOL = m_pCurrentWindow->m_cRealBorderColor.col();
-        static auto *const PBORDERSIZE = &g_pConfigManager->getConfigValuePtr("general:border_size")->intValue;
-        BORDERCOL.a *= alpha / 255.f;
-        renderBorder(pBox, BORDERCOL, *PBORDERSIZE, round);
-        glStencilMask(-1);
-        glStencilFunc(GL_ALWAYS, 1, 0xFF);
-        glDisable(GL_STENCIL_TEST);
-    }
 }
 
 // This probably isn't the fastest
@@ -566,14 +535,14 @@ CFramebuffer* CHyprOpenGLImpl::blurMainFramebufferWithDamage(float a, wlr_box* p
     return currentRenderToFB;
 }
 
-void CHyprOpenGLImpl::renderTextureWithBlur(const CTexture& tex, wlr_box* pBox, float a, wlr_surface* pSurface, int round, bool border) {
+void CHyprOpenGLImpl::renderTextureWithBlur(const CTexture& tex, wlr_box* pBox, float a, wlr_surface* pSurface, int round) {
     RASSERT(m_RenderData.pMonitor, "Tried to render texture with blur without begin()!");
 
     static auto *const PBLURENABLED = &g_pConfigManager->getConfigValuePtr("decoration:blur")->intValue;
     static auto* const PNOBLUROVERSIZED = &g_pConfigManager->getConfigValuePtr("decoration:no_blur_on_oversized")->intValue;
 
     if (*PBLURENABLED == 0 || (*PNOBLUROVERSIZED && m_RenderData.primarySurfaceUVTopLeft != Vector2D(-1, -1)) || (m_pCurrentWindow && m_pCurrentWindow->m_sAdditionalConfigData.forceNoBlur)) {
-        renderTexture(tex, pBox, a, round, false, border, true);
+        renderTexture(tex, pBox, a, round, false, true);
         return;
     }
 
@@ -596,7 +565,7 @@ void CHyprOpenGLImpl::renderTextureWithBlur(const CTexture& tex, wlr_box* pBox,
     }
 
     if (!pixman_region32_not_empty(&inverseOpaque)) {
-        renderTexture(tex, pBox, a, round, false, border); // reject blurring a fully opaque window
+        renderTexture(tex, pBox, a, round, false); // reject blurring a fully opaque window
         return;
     }
 
@@ -630,7 +599,7 @@ void CHyprOpenGLImpl::renderTextureWithBlur(const CTexture& tex, wlr_box* pBox,
     if (pixman_region32_not_empty(&damage)) {
         // render our great blurred FB
         static auto *const PBLURIGNOREOPACITY = &g_pConfigManager->getConfigValuePtr("decoration:blur_ignore_opacity")->intValue;
-        renderTextureInternalWithDamage(POUTFB->m_cTex, &MONITORBOX, *PBLURIGNOREOPACITY ? 255.f : a, &damage, 0, false, false, false, true);
+        renderTextureInternalWithDamage(POUTFB->m_cTex, &MONITORBOX, *PBLURIGNOREOPACITY ? 255.f : a, &damage, 0, false, false, true);
 
         // render the window, but clear stencil
         glClearStencil(0);
@@ -638,36 +607,11 @@ void CHyprOpenGLImpl::renderTextureWithBlur(const CTexture& tex, wlr_box* pBox,
 
         // draw window
         glDisable(GL_STENCIL_TEST);
-        renderTextureInternalWithDamage(tex, pBox, a, &damage, round, false, false, true, true);
-        glEnable(GL_STENCIL_TEST);
-
-        // prep stencil for border
-        glStencilFunc(GL_ALWAYS, 1, -1);
-        glStencilOp(GL_KEEP, GL_KEEP, GL_REPLACE);
-
-        if (border) {
-            // hacky fix to fix broken borders.
-            // TODO: this is kinda slow... question mark?
-            renderRectWithDamage(pBox, CColor(0,0,0,0), &damage, round);
-        }
-
-        // then stop
-        glStencilFunc(GL_EQUAL, 1, -1);
-        glStencilOp(GL_KEEP, GL_KEEP, GL_REPLACE);
+        renderTextureInternalWithDamage(tex, pBox, a, &damage, round, false, false, true);
     }
 
-    // disable the stencil (if no border), finalize everything
-    if (!border) {
-        glStencilMask(-1);
-        glStencilFunc(GL_ALWAYS, 1, 0xFF);
-    } else {
-        auto BORDERCOL = m_pCurrentWindow->m_cRealBorderColor.col();
-        BORDERCOL.a *= a / 255.f;
-        static auto *const PBORDERSIZE = &g_pConfigManager->getConfigValuePtr("general:border_size")->intValue;
-        renderBorder(pBox, BORDERCOL, *PBORDERSIZE, round);
-    }
-    
-    glDisable(GL_STENCIL_TEST);
+    glStencilMask(-1);
+    glStencilFunc(GL_ALWAYS, 1, 0xFF);
     pixman_region32_fini(&damage);
     scissor((wlr_box*)nullptr);
 }
@@ -679,23 +623,67 @@ void pushVert2D(float x, float y, float* arr, int& counter, wlr_box* box) {
     counter++;
 }
 
-void CHyprOpenGLImpl::renderBorder(wlr_box* box, const CColor& col, int thick, int round) {
+void CHyprOpenGLImpl::renderBorder(wlr_box* box, const CColor& col, int round) {
     RASSERT((box->width > 0 && box->height > 0), "Tried to render rect with width/height < 0!");
     RASSERT(m_RenderData.pMonitor, "Tried to render rect without begin()!");
 
-    // this method assumes a set stencil and scaled box
-    box->x -= thick;
-    box->y -= thick;
-    box->width += 2 * thick;
-    box->height += 2 * thick;
+    static auto *const PBORDERSIZE = &g_pConfigManager->getConfigValuePtr("general:border_size")->intValue;
+    static auto *const PMULTISAMPLE = &g_pConfigManager->getConfigValuePtr("decoration:multisample_edges")->intValue;
+
+    // adjust box
+    box->x -= *PBORDERSIZE;
+    box->y -= *PBORDERSIZE;
+    box->width += 2 * *PBORDERSIZE;
+    box->height += 2 * *PBORDERSIZE;
+
+    round += *PBORDERSIZE;
+
+    float matrix[9];
+    wlr_matrix_project_box(matrix, box, wlr_output_transform_invert(!m_bEndFrame ? WL_OUTPUT_TRANSFORM_NORMAL : m_RenderData.pMonitor->transform), 0, m_RenderData.pMonitor->output->transform_matrix);  // TODO: write own, don't use WLR here
+
+    float glMatrix[9];
+    wlr_matrix_multiply(glMatrix, m_RenderData.projection, matrix);
+    wlr_matrix_multiply(glMatrix, matrixFlip180, glMatrix);
+
+    wlr_matrix_transpose(glMatrix, glMatrix);
+
+    glEnable(GL_BLEND);
+    glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
+
+    glUseProgram(m_shBORDER1.program);
 
-    round += thick; // cuz yeah
+    glUniformMatrix3fv(m_shBORDER1.proj, 1, GL_FALSE, glMatrix);
+    glUniform4f(glGetUniformLocation(m_shBORDER1.program, "color"), col.r / 255.f, col.g / 255.f, col.b / 255.f, col.a / 255.f);
 
-    // only draw on non-stencild.
-    glStencilFunc(GL_NOTEQUAL, 1, -1);
+    const auto TOPLEFT = Vector2D(round, round);
+    const auto BOTTOMRIGHT = Vector2D(box->width - round, box->height - round);
+    const auto FULLSIZE = Vector2D(box->width, box->height);
 
-    // draw a rounded rect
-    renderRect(box, col, round);
+    glUniform2f(glGetUniformLocation(m_shBORDER1.program, "topLeft"), (float)TOPLEFT.x, (float)TOPLEFT.y);
+    glUniform2f(glGetUniformLocation(m_shBORDER1.program, "bottomRight"), (float)BOTTOMRIGHT.x, (float)BOTTOMRIGHT.y);
+    glUniform2f(glGetUniformLocation(m_shBORDER1.program, "fullSize"), (float)FULLSIZE.x, (float)FULLSIZE.y);
+    glUniform1f(glGetUniformLocation(m_shBORDER1.program, "radius"), round);
+    glUniform1f(glGetUniformLocation(m_shBORDER1.program, "thick"), *PBORDERSIZE);
+    glUniform1i(glGetUniformLocation(m_shBORDER1.program, "primitiveMultisample"), *PMULTISAMPLE);
+
+    glVertexAttribPointer(m_shBORDER1.posAttrib, 2, GL_FLOAT, GL_FALSE, 0, fullVerts);
+    glVertexAttribPointer(m_shBORDER1.texAttrib, 2, GL_FLOAT, GL_FALSE, 0, fullVerts);
+
+    glEnableVertexAttribArray(m_shBORDER1.posAttrib);
+    glEnableVertexAttribArray(m_shBORDER1.texAttrib);
+
+    if (pixman_region32_not_empty(m_RenderData.pDamage)) {
+        PIXMAN_DAMAGE_FOREACH(m_RenderData.pDamage) {
+            const auto RECT = RECTSARR[i];
+            scissor(&RECT);
+            glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
+        }
+    }
+
+    glDisableVertexAttribArray(m_shBORDER1.posAttrib);
+    glDisableVertexAttribArray(m_shBORDER1.texAttrib);
+
+    glBlendFunc(GL_ONE, GL_ONE_MINUS_SRC_ALPHA);
 }
 
 void CHyprOpenGLImpl::makeWindowSnapshot(CWindow* pWindow) {
diff --git a/src/render/OpenGL.hpp b/src/render/OpenGL.hpp
index 9d6e098d..a70e382b 100644
--- a/src/render/OpenGL.hpp
+++ b/src/render/OpenGL.hpp
@@ -59,9 +59,10 @@ public:
     void    renderRect(wlr_box*, const CColor&, int round = 0);
     void    renderRectWithDamage(wlr_box*, const CColor&, pixman_region32_t* damage, int round = 0);
     void    renderTexture(wlr_texture*, wlr_box*, float a, int round = 0);
-    void    renderTexture(const CTexture&, wlr_box*, float a, int round = 0, bool discardOpaque = false, bool border = false, bool allowPrimary = false);
-    void    renderTextureWithBlur(const CTexture&, wlr_box*, float a, wlr_surface* pSurface, int round = 0, bool border = false);
+    void    renderTexture(const CTexture&, wlr_box*, float a, int round = 0, bool discardOpaque = false, bool allowPrimary = false);
+    void    renderTextureWithBlur(const CTexture&, wlr_box*, float a, wlr_surface* pSurface, int round = 0);
     void    renderRoundedShadow(wlr_box*, int round, int range, float a = 1.0);
+    void    renderBorder(wlr_box*, const CColor&, int round);
 
     void    makeWindowSnapshot(CWindow*);
     void    makeLayerSnapshot(SLayerSurface*);
@@ -108,6 +109,7 @@ private:
     CShader                 m_shBLUR1;
     CShader                 m_shBLUR2;
     CShader                 m_shSHADOW;
+    CShader                 m_shBORDER1;
     //
 
     GLuint                  createProgram(const std::string&, const std::string&);
@@ -117,8 +119,7 @@ private:
     // returns the out FB, can be either Mirror or MirrorSwap
     CFramebuffer*           blurMainFramebufferWithDamage(float a, wlr_box* pBox, pixman_region32_t* damage);
 
-    void                    renderTextureInternalWithDamage(const CTexture&, wlr_box* pBox, float a, pixman_region32_t* damage, int round = 0, bool discardOpaque = false, bool border = false, bool noAA = false, bool allowPrimary = false);
-    void                    renderBorder(wlr_box*, const CColor&, int thick = 1, int round = 0);
+    void                    renderTextureInternalWithDamage(const CTexture&, wlr_box* pBox, float a, pixman_region32_t* damage, int round = 0, bool discardOpaque = false, bool noAA = false, bool allowPrimary = false);
 };
 
 inline std::unique_ptr<CHyprOpenGLImpl> g_pHyprOpenGL;
\ No newline at end of file
diff --git a/src/render/Renderer.cpp b/src/render/Renderer.cpp
index f7328997..5b56dfaa 100644
--- a/src/render/Renderer.cpp
+++ b/src/render/Renderer.cpp
@@ -27,7 +27,13 @@ void renderSurface(struct wlr_surface* surface, int x, int y, void* data) {
 
     if (RDATA->surface && surface == RDATA->surface) {
         g_pHyprOpenGL->m_RenderData.renderingPrimarySurface = true;
-        g_pHyprOpenGL->renderTextureWithBlur(TEXTURE, &windowBox, RDATA->fadeAlpha * RDATA->alpha, surface, rounding, RDATA->decorate);
+        g_pHyprOpenGL->renderTextureWithBlur(TEXTURE, &windowBox, RDATA->fadeAlpha * RDATA->alpha, surface, rounding);
+
+        if (RDATA->decorate) {
+            auto col = g_pHyprOpenGL->m_pCurrentWindow->m_cRealBorderColor.col();
+            col.a *= RDATA->fadeAlpha * RDATA->alpha / 255.f;
+            g_pHyprOpenGL->renderBorder(&windowBox, col, rounding);
+        }
     }
     else
         g_pHyprOpenGL->renderTexture(TEXTURE, &windowBox, RDATA->fadeAlpha * RDATA->alpha, rounding, false, false);
diff --git a/src/render/Shaders.hpp b/src/render/Shaders.hpp
index ac7f9424..1849c621 100644
--- a/src/render/Shaders.hpp
+++ b/src/render/Shaders.hpp
@@ -1,4 +1,5 @@
 #pragma once
 
 #include "shaders/Textures.hpp"
-#include "shaders/Shadow.hpp"
\ No newline at end of file
+#include "shaders/Shadow.hpp"
+#include "shaders/Border.hpp"
\ No newline at end of file
diff --git a/src/render/shaders/Border.hpp b/src/render/shaders/Border.hpp
new file mode 100644
index 00000000..69f1ad5b
--- /dev/null
+++ b/src/render/shaders/Border.hpp
@@ -0,0 +1,91 @@
+#pragma once
+
+#include <string>
+
+// makes a stencil without corners
+inline const std::string FRAGBORDER1 = R"#(
+precision mediump float;
+varying vec4 v_color;
+varying vec2 v_texcoord;
+
+uniform vec2 topLeft;
+uniform vec2 bottomRight;
+uniform vec2 fullSize;
+uniform float radius;
+uniform float thick;
+uniform int primitiveMultisample;
+
+float getOpacityForPixAndCorner(vec2 pix, vec2 corner) {
+
+    if (primitiveMultisample == 0) {
+        float dis = distance(pix + vec2(0.5, 0.5), corner);
+        return dis < radius && dis > radius - thick ? 1.0 : 0.0;
+    }
+
+    float distance1 = distance(pix + vec2(0.25, 0.25), corner);
+    float distance2 = distance(pix + vec2(0.75, 0.25), corner);
+    float distance3 = distance(pix + vec2(0.25, 0.75), corner);
+    float distance4 = distance(pix + vec2(0.75, 0.75), corner);
+
+    float v1 = distance1 < radius && distance1 > radius - thick ? 1.0 : 0.0;
+    float v2 = distance2 < radius && distance2 > radius - thick ? 1.0 : 0.0;
+    float v3 = distance3 < radius && distance3 > radius - thick ? 1.0 : 0.0;
+    float v4 = distance4 < radius && distance4 > radius - thick ? 1.0 : 0.0;
+
+    return (v1 + v2 + v3 + v4) / 4.0;
+}
+
+void main() {
+
+    vec2 pixCoord = fullSize * v_texcoord;
+
+    vec4 pixColor = v_color;
+
+    bool done = false;
+    
+    // check for edges
+    if (pixCoord[0] < topLeft[0]) {
+        if (pixCoord[1] < topLeft[1]) {
+            // top left
+            pixColor[3] = pixColor[3] * getOpacityForPixAndCorner(pixCoord, topLeft + vec2(1,1));
+            done = true;
+        } else if (pixCoord[1] > bottomRight[1]) {
+            // bottom left
+            pixColor[3] = pixColor[3] * getOpacityForPixAndCorner(pixCoord, vec2(topLeft[0] + 1.0, bottomRight[1]));
+            done = true;
+        }
+    } else if (pixCoord[0] > bottomRight[0]) {
+        if (pixCoord[1] < topLeft[1]) {
+            // top right
+            pixColor[3] = pixColor[3] * getOpacityForPixAndCorner(pixCoord, vec2(bottomRight[0], topLeft[1] + 1.0));
+            done = true;
+        } else if (pixCoord[1] > bottomRight[1]) {
+            // bottom right
+            pixColor[3] = pixColor[3] * getOpacityForPixAndCorner(pixCoord, bottomRight);
+            done = true;
+        }
+    }
+
+    // now check for other shit
+    if (!done) {
+        // distance to all straight bb borders
+        float distanceT = pixCoord[1];
+        float distanceB = fullSize[1] - pixCoord[1];
+        float distanceL = pixCoord[0];
+        float distanceR = fullSize[0] - pixCoord[0];
+
+        // get the smallest
+        float smallest = min(min(distanceT, distanceB), min(distanceL, distanceR));
+
+        if (smallest > thick) {
+            discard; return;
+        }
+    }
+
+    if (pixColor[3] == 0.0) {
+        discard; return;
+    }
+
+    gl_FragColor = pixColor;
+}
+)#";
\ No newline at end of file