Move stencil into each framebuffer (#156)

* Move stencil into each framebuffer

Also fixes the stencil being added to the wrong framebuffer

* Initialize texture members on framebuffer init

* removed bind arg

* renamed init to create, changed existing create to update

* moved stencil buffer creation to new function

* removed some now misleading comments

---------

Co-authored-by: William McKinnon <contact@willmckinnon.com>

include/sway/desktop/fx_renderer/fx_framebuffer.h

@@ -10,11 +10,16 @@
 struct fx_framebuffer {
 	struct fx_texture texture;
 	GLuint fb;
+	GLuint stencil_buffer;
 };
 
+struct fx_framebuffer fx_framebuffer_create();
+
 void fx_framebuffer_bind(struct fx_framebuffer *buffer);
 
-void fx_framebuffer_create(struct fx_framebuffer *buffer, int width, int height, bool bind);
+void fx_framebuffer_update(struct fx_framebuffer *buffer, int width, int height);
+
+void fx_framebuffer_add_stencil_buffer(struct fx_framebuffer *buffer, int width, int height);
 
 void fx_framebuffer_release(struct fx_framebuffer *buffer);
 

include/sway/desktop/fx_renderer/fx_renderer.h

@@ -119,8 +119,6 @@ struct fx_renderer {
 
 	int viewport_width, viewport_height;
 
-	GLuint stencil_buffer_id;
-
 	struct fx_framebuffer wlr_buffer; // Just the framebuffer used by wlroots
 	struct fx_framebuffer main_buffer; // The main FB used for rendering
 	struct fx_framebuffer blur_buffer; // Contains the blurred background for tiled windows

sway/desktop/fx_renderer/fx_framebuffer.c

@@ -1,14 +1,24 @@
 #include "log.h"
 #include "sway/desktop/fx_renderer/fx_framebuffer.h"
 
+struct fx_framebuffer fx_framebuffer_create() {
+	return (struct fx_framebuffer) {
+		.fb = -1,
+		.stencil_buffer = -1,
+		.texture.id = 0,
+		.texture.target = 0,
+		.texture.width = -1,
+		.texture.height = -1,
+	};
+}
+
 void fx_framebuffer_bind(struct fx_framebuffer *buffer) {
 	glBindFramebuffer(GL_FRAMEBUFFER, buffer->fb);
 }
 
-void fx_framebuffer_create(struct fx_framebuffer *buffer, int width, int height, bool bind) {
+void fx_framebuffer_update(struct fx_framebuffer *buffer, int width, int height) {
 	bool firstAlloc = false;
 
-	// Create a new framebuffer
 	if (buffer->fb == (uint32_t) -1) {
 		glGenFramebuffers(1, &buffer->fb);
 		firstAlloc = true;
@@ -44,19 +54,38 @@ void fx_framebuffer_create(struct fx_framebuffer *buffer, int width, int height,
 		sway_log(SWAY_DEBUG, "Framebuffer created, status %i", status);
 	}
 
-	// Bind the default framebuffer
 	glBindTexture(GL_TEXTURE_2D, 0);
-	if (bind) {
+}
-		fx_framebuffer_bind(buffer);
+
+void fx_framebuffer_add_stencil_buffer(struct fx_framebuffer *buffer, int width, int height) {
+	if (buffer->stencil_buffer == (uint32_t) -1) {
+		glGenRenderbuffers(1, &buffer->stencil_buffer);
+		glBindRenderbuffer(GL_RENDERBUFFER, buffer->stencil_buffer);
+		glRenderbufferStorage(GL_RENDERBUFFER, GL_STENCIL_INDEX8, width, height);
+		glFramebufferRenderbuffer(GL_FRAMEBUFFER, GL_STENCIL_ATTACHMENT, GL_RENDERBUFFER, buffer->stencil_buffer);
+		GLenum status = glCheckFramebufferStatus(GL_FRAMEBUFFER);
+		if (status != GL_FRAMEBUFFER_COMPLETE) {
+			sway_log(SWAY_ERROR, "Stencil buffer incomplete, couldn't create! (FB status: %i)", status);
+			return;
+		}
+		sway_log(SWAY_DEBUG, "Stencil buffer created, status %i", status);
 	}
 }
 
 void fx_framebuffer_release(struct fx_framebuffer *buffer) {
+	// Release the framebuffer
 	if (buffer->fb != (uint32_t) -1 && buffer->fb) {
 		glDeleteFramebuffers(1, &buffer->fb);
 	}
 	buffer->fb= -1;
 
+	// Release the stencil buffer
+	if (buffer->stencil_buffer != (uint32_t)-1 && buffer->stencil_buffer) {
+		glDeleteRenderbuffers(1, &buffer->stencil_buffer);
+	}
+	buffer->stencil_buffer = -1;
+
+	// Release the texture
 	if (buffer->texture.id) {
 		glDeleteTextures(1, &buffer->texture.id);
 	}

sway/desktop/fx_renderer/fx_renderer.c

@@ -13,6 +13,7 @@
 #include <wlr/util/box.h>
 
 #include "log.h"
+#include "sway/desktop/fx_renderer/fx_framebuffer.h"
 #include "sway/desktop/fx_renderer/fx_renderer.h"
 #include "sway/desktop/fx_renderer/matrix.h"
 #include "sway/server.h"
@@ -35,30 +36,6 @@ static const GLfloat verts[] = {
 	0, 1, // bottom left
 };
 
-static void create_stencil_buffer(GLuint *buffer_id, int width, int height) {
-	if (*buffer_id != (uint32_t) -1) {
-		return;
-	}
-
-	glGenRenderbuffers(1, buffer_id);
-	glBindRenderbuffer(GL_RENDERBUFFER, *buffer_id);
-	glRenderbufferStorage(GL_RENDERBUFFER, GL_STENCIL_INDEX8, width, height);
-	glFramebufferRenderbuffer(GL_FRAMEBUFFER, GL_STENCIL_ATTACHMENT, GL_RENDERBUFFER, *buffer_id);
-	GLenum status = glCheckFramebufferStatus(GL_FRAMEBUFFER);
-	if (status != GL_FRAMEBUFFER_COMPLETE) {
-		sway_log(SWAY_ERROR, "Stencilbuffer incomplete, couldn't create! (FB status: %i)", status);
-		return;
-	}
-	sway_log(SWAY_DEBUG, "Stencilbuffer created, status %i", status);
-}
-
-static void release_stencil_buffer(GLuint *buffer_id) {
-	if (*buffer_id != (uint32_t)-1 && buffer_id) {
-		glDeleteRenderbuffers(1, buffer_id);
-	}
-	*buffer_id = -1;
-}
-
 static GLuint compile_shader(GLuint type, const GLchar *src) {
 	GLuint shader = glCreateShader(type);
 	glShaderSource(shader, 1, &src, NULL);
@@ -288,12 +265,10 @@ struct fx_renderer *fx_renderer_create(struct wlr_egl *egl) {
 		return NULL;
 	}
 
-	renderer->main_buffer.fb = -1;
+	renderer->main_buffer = fx_framebuffer_create();
-
+	renderer->blur_buffer = fx_framebuffer_create();
-	renderer->blur_buffer.fb = -1;
+	renderer->effects_buffer = fx_framebuffer_create();
-	renderer->effects_buffer.fb = -1;
+	renderer->effects_buffer_swapped = fx_framebuffer_create();
-	renderer->effects_buffer_swapped.fb = -1;
-	renderer->stencil_buffer_id = -1;
 
 	renderer->blur_buffer_dirty = true;
 
@@ -414,7 +389,6 @@ void fx_renderer_fini(struct fx_renderer *renderer) {
 	fx_framebuffer_release(&renderer->blur_buffer);
 	fx_framebuffer_release(&renderer->effects_buffer);
 	fx_framebuffer_release(&renderer->effects_buffer_swapped);
-	release_stencil_buffer(&renderer->stencil_buffer_id);
 }
 
 void fx_renderer_begin(struct fx_renderer *renderer, int width, int height) {
@@ -432,10 +406,13 @@ void fx_renderer_begin(struct fx_renderer *renderer, int width, int height) {
 	renderer->wlr_buffer.fb = wlr_fb;
 
 	// Create the framebuffers
-	fx_framebuffer_create(&renderer->main_buffer, width, height, true);
+	fx_framebuffer_update(&renderer->main_buffer, width, height);
-	fx_framebuffer_create(&renderer->effects_buffer, width, height, false);
+	fx_framebuffer_update(&renderer->effects_buffer, width, height);
-	fx_framebuffer_create(&renderer->effects_buffer_swapped, width, height, false);
+	fx_framebuffer_update(&renderer->effects_buffer_swapped, width, height);
-	create_stencil_buffer(&renderer->stencil_buffer_id, width, height);
+
+	// Add a stencil buffer to the main buffer & bind the main buffer
+	fx_framebuffer_add_stencil_buffer(&renderer->main_buffer, width, height);
+	fx_framebuffer_bind(&renderer->main_buffer);
 
 	// refresh projection matrix
 	matrix_projection(renderer->projection, width, height,
@@ -450,7 +427,6 @@ void fx_renderer_begin(struct fx_renderer *renderer, int width, int height) {
 void fx_renderer_end(struct fx_renderer *renderer) {
 	// Release the main buffer
 	fx_framebuffer_release(&renderer->main_buffer);
-	release_stencil_buffer(&renderer->stencil_buffer_id);
 }
 
 void fx_renderer_clear(const float color[static 4]) {

sway/desktop/render.c

@@ -494,8 +494,10 @@ void render_output_blur(struct sway_output *output, pixman_region32_t *damage) {
 	struct fx_framebuffer *buffer = get_main_buffer_blur(renderer, output, &fake_damage, &monitor_box);
 
 	// Render the newly blurred content into the blur_buffer
-	fx_framebuffer_create(&renderer->blur_buffer,
+	fx_framebuffer_update(&renderer->blur_buffer,
-			output->renderer->viewport_width, output->renderer->viewport_height, true);
+			output->renderer->viewport_width, output->renderer->viewport_height);
+	fx_framebuffer_bind(&renderer->blur_buffer);
+
 	// Clear the damaged region of the blur_buffer
 	float clear_color[] = { 0, 0, 0, 0 };
 	int nrects;