diff --git a/include/sway/input/seat.h b/include/sway/input/seat.h
index eb0d1a02..e9f553f3 100644
--- a/include/sway/input/seat.h
+++ b/include/sway/input/seat.h
@@ -52,7 +52,6 @@ struct sway_seat {
 	bool has_focus;
 	struct wl_list focus_stack; // list of containers in focus order
 	struct sway_workspace *workspace;
-	struct sway_node *prev_focus;
 
 	// If the focused layer is set, views cannot receive keyboard focus
 	struct wlr_layer_surface_v1 *focused_layer;
diff --git a/sway/input/seat.c b/sway/input/seat.c
index 08b2e7cf..cffceaae 100644
--- a/sway/input/seat.c
+++ b/sway/input/seat.c
@@ -646,7 +646,6 @@ void seat_set_focus(struct sway_seat *seat, struct sway_node *node) {
 	}
 
 	struct sway_node *last_focus = seat_get_focus(seat);
-	seat->prev_focus = last_focus;
 	if (last_focus == node) {
 		return;
 	}
@@ -1190,12 +1189,17 @@ void seat_pointer_notify_button(struct sway_seat *seat, uint32_t time_msec,
 
 void seat_consider_warp_to_focus(struct sway_seat *seat) {
 	struct sway_node *focus = seat_get_focus(seat);
-	if (config->mouse_warping == WARP_NO || !focus || !seat->prev_focus) {
+	if (config->mouse_warping == WARP_NO || !focus) {
 		return;
 	}
-	if (config->mouse_warping == WARP_OUTPUT &&
-			node_get_output(focus) == node_get_output(seat->prev_focus)) {
-		return;
+	if (config->mouse_warping == WARP_OUTPUT) {
+		struct sway_output *output = node_get_output(focus);
+		struct wlr_box box;
+		output_get_box(output, &box);
+		if (wlr_box_contains_point(&box,
+					seat->cursor->cursor->x, seat->cursor->cursor->y)) {
+			return;
+		}
 	}
 
 	if (focus->type == N_CONTAINER) {