fix: update stencil if the output size changes (#178)

include/sway/desktop/fx_renderer/fx_framebuffer.h

@@ -5,12 +5,13 @@
 #include <stdbool.h>
 #include <wlr/types/wlr_output.h>
 
+#include "sway/desktop/fx_renderer/fx_stencilbuffer.h"
 #include "sway/desktop/fx_renderer/fx_texture.h"
 
 struct fx_framebuffer {
-	struct fx_texture texture;
 	GLuint fb;
-	GLuint stencil_buffer;
+	struct fx_stencilbuffer stencil_buffer;
+	struct fx_texture texture;
 };
 
 struct fx_framebuffer fx_framebuffer_create();
@@ -23,5 +24,4 @@ void fx_framebuffer_add_stencil_buffer(struct fx_framebuffer *buffer, int width,
 
 void fx_framebuffer_release(struct fx_framebuffer *buffer);
 
-
 #endif

include/sway/desktop/fx_renderer/fx_stencilbuffer.h

@@ -0,0 +1,18 @@
+#ifndef FX_STENCILBUFFER_H
+#define FX_STENCILBUFFER_H
+
+#include <GLES2/gl2.h>
+#include <stdbool.h>
+#include <wlr/render/wlr_texture.h>
+
+struct fx_stencilbuffer {
+	GLuint rb;
+	int width;
+	int height;
+};
+
+struct fx_stencilbuffer fx_stencilbuffer_create();
+
+void fx_stencilbuffer_release(struct fx_stencilbuffer *stencil_buffer);
+
+#endif

include/sway/desktop/fx_renderer/fx_texture.h

@@ -13,6 +13,10 @@ struct fx_texture {
 	int height;
 };
 
+struct fx_texture fx_texture_create();
+
 struct fx_texture fx_texture_from_wlr_texture(struct wlr_texture *tex);
 
+void fx_texture_release(struct fx_texture *texture);
+
 #endif

sway/desktop/fx_renderer/fx_framebuffer.c

@@ -1,14 +1,13 @@
 #include "log.h"
 #include "sway/desktop/fx_renderer/fx_framebuffer.h"
+#include "sway/desktop/fx_renderer/fx_stencilbuffer.h"
+#include "sway/desktop/fx_renderer/fx_texture.h"
 
 struct fx_framebuffer fx_framebuffer_create() {
 	return (struct fx_framebuffer) {
 		.fb = -1,
-		.stencil_buffer = -1,
-		.texture.id = 0,
-		.texture.target = 0,
-		.texture.width = -1,
-		.texture.height = -1,
+		.stencil_buffer = fx_stencilbuffer_create(),
+		.texture = fx_texture_create(),
 	};
 }
 
@@ -17,15 +16,15 @@ void fx_framebuffer_bind(struct fx_framebuffer *buffer) {
 }
 
 void fx_framebuffer_update(struct fx_framebuffer *buffer, int width, int height) {
-	bool firstAlloc = false;
+	bool first_alloc = false;
 
 	if (buffer->fb == (uint32_t) -1) {
 		glGenFramebuffers(1, &buffer->fb);
-		firstAlloc = true;
+		first_alloc = true;
 	}
 
 	if (buffer->texture.id == 0) {
-		firstAlloc = true;
+		first_alloc = true;
 		glGenTextures(1, &buffer->texture.id);
 		glBindTexture(GL_TEXTURE_2D, buffer->texture.id);
 		glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
@@ -34,7 +33,7 @@ void fx_framebuffer_update(struct fx_framebuffer *buffer, int width, int height)
 		glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
 	}
 
-	if (firstAlloc || buffer->texture.width != width || buffer->texture.height != height) {
+	if (first_alloc || buffer->texture.width != width || buffer->texture.height != height) {
 		glBindTexture(GL_TEXTURE_2D, buffer->texture.id);
 		glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, width, height, 0, GL_RGBA, GL_UNSIGNED_BYTE, 0);
 
@@ -58,11 +57,20 @@ void fx_framebuffer_update(struct fx_framebuffer *buffer, int width, int height)
 }
 
 void fx_framebuffer_add_stencil_buffer(struct fx_framebuffer *buffer, int width, int height) {
-	if (buffer->stencil_buffer == (uint32_t) -1) {
-		glGenRenderbuffers(1, &buffer->stencil_buffer);
-		glBindRenderbuffer(GL_RENDERBUFFER, buffer->stencil_buffer);
+	bool first_alloc = false;
+
+	if (buffer->stencil_buffer.rb == (uint32_t) -1) {
+		glGenRenderbuffers(1, &buffer->stencil_buffer.rb);
+		first_alloc = true;
+	}
+
+	if (first_alloc || buffer->stencil_buffer.width != width || buffer->stencil_buffer.height != height) {
+		glBindRenderbuffer(GL_RENDERBUFFER, buffer->stencil_buffer.rb);
 		glRenderbufferStorage(GL_RENDERBUFFER, GL_STENCIL_INDEX8, width, height);
-		glFramebufferRenderbuffer(GL_FRAMEBUFFER, GL_STENCIL_ATTACHMENT, GL_RENDERBUFFER, buffer->stencil_buffer);
+		glFramebufferRenderbuffer(GL_FRAMEBUFFER, GL_STENCIL_ATTACHMENT, GL_RENDERBUFFER, buffer->stencil_buffer.rb);
+		buffer->stencil_buffer.width = width;
+		buffer->stencil_buffer.height = height;
+
 		GLenum status = glCheckFramebufferStatus(GL_FRAMEBUFFER);
 		if (status != GL_FRAMEBUFFER_COMPLETE) {
 			sway_log(SWAY_ERROR, "Stencil buffer incomplete, couldn't create! (FB status: %i)", status);
@@ -80,16 +88,8 @@ void fx_framebuffer_release(struct fx_framebuffer *buffer) {
 	buffer->fb = -1;
 
 	// Release the stencil buffer
-	if (buffer->stencil_buffer != (uint32_t)-1 && buffer->stencil_buffer) {
-		glDeleteRenderbuffers(1, &buffer->stencil_buffer);
-	}
-	buffer->stencil_buffer = -1;
+	fx_stencilbuffer_release(&buffer->stencil_buffer);
 
 	// Release the texture
-	if (buffer->texture.id) {
-		glDeleteTextures(1, &buffer->texture.id);
-	}
-	buffer->texture.id = 0;
-	buffer->texture.width = -1;
-	buffer->texture.height = -1;
+	fx_texture_release(&buffer->texture);
 }

sway/desktop/fx_renderer/fx_stencilbuffer.c

@@ -0,0 +1,21 @@
+#include <assert.h>
+#include <wlr/render/gles2.h>
+
+#include "sway/desktop/fx_renderer/fx_stencilbuffer.h"
+
+struct fx_stencilbuffer fx_stencilbuffer_create() {
+	return (struct fx_stencilbuffer) {
+		.rb = -1,
+		.width = -1,
+		.height = -1,
+	};
+}
+
+void fx_stencilbuffer_release(struct fx_stencilbuffer *stencil_buffer) {
+	if (stencil_buffer->rb != (uint32_t) -1 && stencil_buffer->rb) {
+		glDeleteRenderbuffers(1, &stencil_buffer->rb);
+	}
+	stencil_buffer->rb = -1;
+	stencil_buffer->width = -1;
+	stencil_buffer->height = -1;
+}

sway/desktop/fx_renderer/fx_texture.c

@@ -3,6 +3,15 @@
 
 #include "sway/desktop/fx_renderer/fx_texture.h"
 
+struct fx_texture fx_texture_create() {
+	return (struct fx_texture) {
+		.id = 0,
+		.target = 0,
+		.width = -1,
+		.height = -1,
+	};
+}
+
 struct fx_texture fx_texture_from_wlr_texture(struct wlr_texture *texture) {
 	assert(wlr_texture_is_gles2(texture));
 
@@ -17,3 +26,12 @@ struct fx_texture fx_texture_from_wlr_texture(struct wlr_texture* texture) {
 		.height = texture->height,
 	};
 }
+
+void fx_texture_release(struct fx_texture *texture) {
+	if (texture->id) {
+		glDeleteTextures(1, &texture->id);
+	}
+	texture->id = 0;
+	texture->width = -1;
+	texture->height = -1;
+}

sway/meson.build

@@ -17,6 +17,7 @@ sway_sources = files(
 	'desktop/desktop.c',
 	'desktop/fx_renderer/fx_framebuffer.c',
 	'desktop/fx_renderer/fx_renderer.c',
+	'desktop/fx_renderer/fx_stencilbuffer.c',
 	'desktop/fx_renderer/fx_texture.c',
 	'desktop/fx_renderer/matrix.c',
 	'desktop/idle_inhibit_v1.c',