diff --git a/include/sway/input/seat.h b/include/sway/input/seat.h
index 78dbda6f..77c2278d 100644
--- a/include/sway/input/seat.h
+++ b/include/sway/input/seat.h
@@ -241,7 +241,7 @@ void seatop_begin_default(struct sway_seat *seat);
 void seatop_begin_down(struct sway_seat *seat, struct sway_container *con,
 		uint32_t time_msec, double sx, double sy);
 
-void seatop_begin_down_on_layer_surface(struct sway_seat *seat,
+void seatop_begin_down_on_surface(struct sway_seat *seat,
 		struct wlr_surface *surface, uint32_t time_msec, double sx, double sy);
 
 void seatop_begin_move_floating(struct sway_seat *seat,
diff --git a/sway/input/seatop_default.c b/sway/input/seatop_default.c
index 7a3745d2..4320a3b4 100644
--- a/sway/input/seatop_default.c
+++ b/sway/input/seatop_default.c
@@ -374,7 +374,7 @@ static void handle_button(struct sway_seat *seat, uint32_t time_msec,
 			transaction_commit_dirty();
 		}
 		if (state == WLR_BUTTON_PRESSED) {
-			seatop_begin_down_on_layer_surface(seat, surface, time_msec, sx, sy);
+			seatop_begin_down_on_surface(seat, surface, time_msec, sx, sy);
 		}
 		seat_pointer_notify_button(seat, time_msec, button, state);
 		return;
diff --git a/sway/input/seatop_down.c b/sway/input/seatop_down.c
index cc54b90b..ecc34fea 100644
--- a/sway/input/seatop_down.c
+++ b/sway/input/seatop_down.c
@@ -10,8 +10,10 @@
 
 struct seatop_down_event {
 	struct sway_container *con;
+	struct sway_seat *seat;
+	struct wl_listener surface_destroy;
 	struct wlr_surface *surface;
-	double ref_lx, ref_ly;		 // cursor's x/y at start of op
+	double ref_lx, ref_ly;         // cursor's x/y at start of op
 	double ref_con_lx, ref_con_ly; // container's x/y at start of op
 };
 
@@ -71,6 +73,14 @@ static void handle_tablet_tool_motion(struct sway_seat *seat,
 	}
 }
 
+static void handle_destroy(struct wl_listener *listener, void *data) {
+	struct seatop_down_event *e =
+		wl_container_of(listener, e, surface_destroy);
+	if (e) {
+		seatop_begin_default(e->seat);
+	}
+}
+
 static void handle_unref(struct sway_seat *seat, struct sway_container *con) {
 	struct seatop_down_event *e = seat->seatop_data;
 	if (e->con == con) {
@@ -78,6 +88,11 @@ static void handle_unref(struct sway_seat *seat, struct sway_container *con) {
 	}
 }
 
+static void handle_end(struct sway_seat *seat) {
+	struct seatop_down_event *e = seat->seatop_data;
+	wl_list_remove(&e->surface_destroy.link);
+}
+
 static const struct sway_seatop_impl seatop_impl = {
 	.button = handle_button,
 	.pointer_motion = handle_pointer_motion,
@@ -85,33 +100,21 @@ static const struct sway_seatop_impl seatop_impl = {
 	.tablet_tool_tip = handle_tablet_tool_tip,
 	.tablet_tool_motion = handle_tablet_tool_motion,
 	.unref = handle_unref,
+	.end = handle_end,
 	.allow_set_cursor = true,
 };
 
 void seatop_begin_down(struct sway_seat *seat, struct sway_container *con,
 		uint32_t time_msec, double sx, double sy) {
-	seatop_end(seat);
-
-	struct seatop_down_event *e =
-		calloc(1, sizeof(struct seatop_down_event));
-	if (!e) {
-		return;
-	}
+	seatop_begin_down_on_surface(seat, con->view->surface, time_msec, sx, sy);
+	struct seatop_down_event *e = seat->seatop_data;
 	e->con = con;
-	e->surface = con->view->surface;
-	e->ref_lx = seat->cursor->cursor->x;
-	e->ref_ly = seat->cursor->cursor->y;
-	e->ref_con_lx = sx;
-	e->ref_con_ly = sy;
-
-	seat->seatop_impl = &seatop_impl;
-	seat->seatop_data = e;
 
 	container_raise_floating(con);
 	transaction_commit_dirty();
 }
 
-void seatop_begin_down_on_layer_surface(struct sway_seat *seat,
+void seatop_begin_down_on_surface(struct sway_seat *seat,
 		struct wlr_surface *surface, uint32_t time_msec, double sx, double sy) {
 	seatop_end(seat);
 
@@ -121,7 +124,10 @@ void seatop_begin_down_on_layer_surface(struct sway_seat *seat,
 		return;
 	}
 	e->con = NULL;
+	e->seat = seat;
 	e->surface = surface;
+	wl_signal_add(&e->surface->events.destroy, &e->surface_destroy);
+	e->surface_destroy.notify = handle_destroy;
 	e->ref_lx = seat->cursor->cursor->x;
 	e->ref_ly = seat->cursor->cursor->y;
 	e->ref_con_lx = sx;