Unify rounding shaders

3 files changed, 65 insertions, 74 deletions

diff --git a/src/render/OpenGL.cpp b/src/render/OpenGL.cpp
index 2964b3a1..3345db60 100644
--- a/src/render/OpenGL.cpp
+++ b/src/render/OpenGL.cpp
@@ -809,7 +809,7 @@ void CHyprOpenGLImpl::renderBorder(wlr_box* box, const CColor& col, int round) {
     RASSERT(m_RenderData.pMonitor, "Tried to render rect without begin()!");
 
     if (!pixman_region32_not_empty(m_RenderData.pDamage) || (m_pCurrentWindow && m_pCurrentWindow->m_sAdditionalConfigData.forceNoBorder))
-	return;
+	    return;
 
     static auto *const PBORDERSIZE = &g_pConfigManager->getConfigValuePtr("general:border_size")->intValue;
     static auto *const PMULTISAMPLE = &g_pConfigManager->getConfigValuePtr("decoration:multisample_edges")->intValue;
@@ -860,12 +860,15 @@ void CHyprOpenGLImpl::renderBorder(wlr_box* box, const CColor& col, int round) {
 #endif
     glUniform4f(m_RenderData.pCurrentMonData->m_shBORDER1.color, col.r / 255.f, col.g / 255.f, col.b / 255.f, col.a / 255.f);
 
-    const auto TOPLEFT = Vector2D(round, round);
-    const auto BOTTOMRIGHT = Vector2D(box->width - round, box->height - round);
-    const auto FULLSIZE = Vector2D(box->width, box->height);
+    wlr_box transformedBox;
+    wlr_box_transform(&transformedBox, box, wlr_output_transform_invert(m_RenderData.pMonitor->transform),
+                      m_RenderData.pMonitor->vecTransformedSize.x, m_RenderData.pMonitor->vecTransformedSize.y);
+
+    const auto TOPLEFT = Vector2D(transformedBox.x, transformedBox.y);
+    const auto FULLSIZE = Vector2D(transformedBox.width, transformedBox.height);
 
     glUniform2f(m_RenderData.pCurrentMonData->m_shBORDER1.topLeft, (float)TOPLEFT.x, (float)TOPLEFT.y);
-    glUniform2f(m_RenderData.pCurrentMonData->m_shBORDER1.bottomRight, (float)BOTTOMRIGHT.x, (float)BOTTOMRIGHT.y);
+    //glUniform2f(m_RenderData.pCurrentMonData->m_shBORDER1.bottomRight, (float)BOTTOMRIGHT.x, (float)BOTTOMRIGHT.y);
     glUniform2f(m_RenderData.pCurrentMonData->m_shBORDER1.fullSize, (float)FULLSIZE.x, (float)FULLSIZE.y);
     glUniform1f(m_RenderData.pCurrentMonData->m_shBORDER1.radius, round);
     glUniform1f(m_RenderData.pCurrentMonData->m_shBORDER1.thick, scaledBorderSize);
@@ -890,13 +893,13 @@ void CHyprOpenGLImpl::renderBorder(wlr_box* box, const CColor& col, int round) {
             }
         }
 
-	pixman_region32_fini(&damageClip);
+	    pixman_region32_fini(&damageClip);
     } else {
-	PIXMAN_DAMAGE_FOREACH(m_RenderData.pDamage) {
-	    const auto RECT = RECTSARR[i];
-	    scissor(&RECT);
-	    glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
-	}
+	    PIXMAN_DAMAGE_FOREACH(m_RenderData.pDamage) {
+	        const auto RECT = RECTSARR[i];
+	        scissor(&RECT);
+	        glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
+	    }
     }
 
     glDisableVertexAttribArray(m_RenderData.pCurrentMonData->m_shBORDER1.posAttrib);
diff --git a/src/render/shaders/Border.hpp b/src/render/shaders/Border.hpp
index 806fa64a..6cadede6 100644
--- a/src/render/shaders/Border.hpp
+++ b/src/render/shaders/Border.hpp
@@ -9,82 +9,70 @@ varying vec4 v_color;
 varying vec2 v_texcoord;
 
 uniform vec2 topLeft;
-uniform vec2 bottomRight;
 uniform vec2 fullSize;
 uniform float radius;
 uniform float thick;
 uniform int primitiveMultisample;
 
-float getOpacityForPixAndCorner(vec2 pix, vec2 corner) {
+void main() {
 
-    if (primitiveMultisample == 0) {
-        float dis = distance(pix + vec2(0.5, 0.5), corner);
-        return dis < radius && dis > radius - thick ? 1.0 : 0.0;
-    }
+    highp vec2 pixCoord = vec2(gl_FragCoord);
+    vec2 originalPixCoord = fullSize * v_texcoord;
 
-    float distance1 = distance(pix + vec2(0.25, 0.25), corner);
-    float distance2 = distance(pix + vec2(0.75, 0.25), corner);
-    float distance3 = distance(pix + vec2(0.25, 0.75), corner);
-    float distance4 = distance(pix + vec2(0.75, 0.75), corner);
+    vec4 pixColor = v_color;
 
-    float v1 = distance1 < radius && distance1 > radius - thick ? 1.0 : 0.0;
-    float v2 = distance2 < radius && distance2 > radius - thick ? 1.0 : 0.0;
-    float v3 = distance3 < radius && distance3 > radius - thick ? 1.0 : 0.0;
-    float v4 = distance4 < radius && distance4 > radius - thick ? 1.0 : 0.0;
+    bool done = false;
 
-    return (v1 + v2 + v3 + v4) / 4.0;
-}
+    pixCoord -= topLeft + fullSize * 0.5;
+    pixCoord *= vec2(lessThan(pixCoord, vec2(0.0))) * -2.0 + 1.0;
+    pixCoord -= fullSize * 0.5 - radius;
 
-void main() {
+    if (min(pixCoord.x, pixCoord.y) > 0.0) {
 
-    vec2 pixCoord = fullSize * v_texcoord;
+	    float dist = length(pixCoord);
 
-    vec4 pixColor = v_color;
+	    if (dist > radius || dist < radius - thick - 1.0)
+	        discard;
 
-    bool done = false;
+	    if (primitiveMultisample == 1 && (dist > radius - 1.0 || dist < radius - thick + 1.0)) {
+	        float distances = 0.0;
+            float len = length(pixCoord + vec2(0.25, 0.25));
+	        distances += float(len < radius && len > radius - thick);
+            len = length(pixCoord + vec2(0.75, 0.25));
+            distances += float(len < radius && len > radius - thick);
+            len = length(pixCoord + vec2(0.25, 0.75));
+            distances += float(len < radius && len > radius - thick);
+            len = length(pixCoord + vec2(0.75, 0.75));
+            distances += float(len < radius && len > radius - thick);
 
-    // check for edges
-    if (pixCoord[0] < topLeft[0]) {
-        if (pixCoord[1] < topLeft[1]) {
-            // top left
-            pixColor[3] = pixColor[3] * getOpacityForPixAndCorner(pixCoord, topLeft + vec2(1,1));
-            done = true;
-        } else if (pixCoord[1] > bottomRight[1]) {
-            // bottom left
-            pixColor[3] = pixColor[3] * getOpacityForPixAndCorner(pixCoord, vec2(topLeft[0] + 1.0, bottomRight[1]));
-            done = true;
-        }
-    } else if (pixCoord[0] > bottomRight[0]) {
-        if (pixCoord[1] < topLeft[1]) {
-            // top right
-            pixColor[3] = pixColor[3] * getOpacityForPixAndCorner(pixCoord, vec2(bottomRight[0], topLeft[1] + 1.0));
-            done = true;
-        } else if (pixCoord[1] > bottomRight[1]) {
-            // bottom right
-            pixColor[3] = pixColor[3] * getOpacityForPixAndCorner(pixCoord, bottomRight);
-            done = true;
+	        if (distances == 0.0)
+		        discard;
+
+	        distances /= 4.0;
+
+	        pixColor = pixColor * distances;
         }
+
+        done = true;
     }
 
     // now check for other shit
     if (!done) {
         // distance to all straight bb borders
-        float distanceT = pixCoord[1];
-        float distanceB = fullSize[1] - pixCoord[1];
-        float distanceL = pixCoord[0];
-        float distanceR = fullSize[0] - pixCoord[0];
+        float distanceT = originalPixCoord[1];
+        float distanceB = fullSize[1] - originalPixCoord[1];
+        float distanceL = originalPixCoord[0];
+        float distanceR = fullSize[0] - originalPixCoord[0];
 
         // get the smallest
         float smallest = min(min(distanceT, distanceB), min(distanceL, distanceR));
 
-        if (smallest > thick) {
-            discard; return;
-        }
+        if (smallest > thick)
+            discard;
     }
 
-    if (pixColor[3] == 0.0) {
-        discard; return;
-    }
+    if (pixColor[3] == 0.0)
+        discard;
 
     gl_FragColor = pixColor;
 }
diff --git a/src/render/shaders/Textures.hpp b/src/render/shaders/Textures.hpp
index 3b696a70..053635b2 100644
--- a/src/render/shaders/Textures.hpp
+++ b/src/render/shaders/Textures.hpp
@@ -13,24 +13,24 @@ inline static constexpr auto ROUNDED_SHADER_FUNC = [](const std::string colorVar
 
     if (pixCoord.x + pixCoord.y > radius) {
 
-	float dist = length(pixCoord);
+	    float dist = length(pixCoord);
 
-	if (dist > radius)
-	    discard;
+	    if (dist > radius)
+	        discard;
 
-	if (primitiveMultisample == 1 && dist > radius - 1.0) {
-	    float distances = 0.0;
-	    distances += float(length(pixCoord + vec2(0.25, 0.25)) < radius);
-	    distances += float(length(pixCoord + vec2(0.75, 0.25)) < radius);
-	    distances += float(length(pixCoord + vec2(0.25, 0.75)) < radius);
-	    distances += float(length(pixCoord + vec2(0.75, 0.75)) < radius);
+	    if (primitiveMultisample == 1 && dist > radius - 1.0) {
+	        float distances = 0.0;
+	        distances += float(length(pixCoord + vec2(0.25, 0.25)) < radius);
+	        distances += float(length(pixCoord + vec2(0.75, 0.25)) < radius);
+	        distances += float(length(pixCoord + vec2(0.25, 0.75)) < radius);
+	        distances += float(length(pixCoord + vec2(0.75, 0.75)) < radius);
 
-	    if (distances == 0.0)
-		discard;
+	        if (distances == 0.0)
+		        discard;
 
-	    distances /= 4.0;
+	        distances /= 4.0;
 
-	    )#" + colorVarName + R"#( = )#" + colorVarName + R"#( * distances;
+	        )#" + colorVarName + R"#( = )#" + colorVarName + R"#( * distances;
         }
 
     }