diff --git a/include/sway/input/cursor.h b/include/sway/input/cursor.h
index 5dd109ca..b0a3a7c5 100644
--- a/include/sway/input/cursor.h
+++ b/include/sway/input/cursor.h
@@ -11,6 +11,7 @@ struct sway_cursor {
 	} previous;
 	struct wlr_xcursor_manager *xcursor_manager;
 
+	const char *image;
 	struct wl_client *image_client;
 
 	struct wl_listener motion;
@@ -37,4 +38,7 @@ void cursor_send_pointer_motion(struct sway_cursor *cursor, uint32_t time_msec,
 void dispatch_cursor_button(struct sway_cursor *cursor, uint32_t time_msec,
 	uint32_t button, enum wlr_button_state state);
 
+void cursor_set_image(struct sway_cursor *cursor, const char *image,
+		struct wl_client *client);
+
 #endif
diff --git a/sway/input/cursor.c b/sway/input/cursor.c
index ec83746e..43721d28 100644
--- a/sway/input/cursor.c
+++ b/sway/input/cursor.c
@@ -371,22 +371,16 @@ void cursor_send_pointer_motion(struct sway_cursor *cursor, uint32_t time_msec,
 		// Reset cursor if switching between clients
 		struct wl_client *client = wl_resource_get_client(surface->resource);
 		if (client != cursor->image_client) {
-			wlr_xcursor_manager_set_cursor_image(cursor->xcursor_manager,
-					"left_ptr", cursor->cursor);
-			cursor->image_client = client;
+			cursor_set_image(cursor, "left_ptr", client);
 		}
 	} else if (c && container_is_floating(c)) {
 		// Try a floating container's resize edge
 		enum wlr_edges edge = find_resize_edge(c, cursor);
 		const char *image = edge == WLR_EDGE_NONE ?
 			"left_ptr" : wlr_xcursor_get_resize_name(edge);
-		wlr_xcursor_manager_set_cursor_image(cursor->xcursor_manager, image,
-				cursor->cursor);
-		cursor->image_client = NULL;
+		cursor_set_image(cursor, image, NULL);
 	} else {
-		wlr_xcursor_manager_set_cursor_image(cursor->xcursor_manager,
-				"left_ptr", cursor->cursor);
-		cursor->image_client = NULL;
+		cursor_set_image(cursor, "left_ptr", NULL);
 	}
 
 	// send pointer enter/leave
@@ -725,6 +719,16 @@ static void handle_request_set_cursor(struct wl_listener *listener,
 	cursor->image_client = focused_client;
 }
 
+void cursor_set_image(struct sway_cursor *cursor, const char *image,
+		struct wl_client *client) {
+	if (!cursor->image || strcmp(cursor->image, image) != 0) {
+		wlr_xcursor_manager_set_cursor_image(cursor->xcursor_manager, image,
+				cursor->cursor);
+		cursor->image = image;
+	}
+	cursor->image_client = client;
+}
+
 void sway_cursor_destroy(struct sway_cursor *cursor) {
 	if (!cursor) {
 		return;