diff --git a/sway/input/cursor.c b/sway/input/cursor.c
index dbb8c48d..f5c2bd17 100644
--- a/sway/input/cursor.c
+++ b/sway/input/cursor.c
@@ -378,7 +378,6 @@ void dispatch_cursor_button(struct sway_cursor *cursor,
 static void handle_pointer_button(struct wl_listener *listener, void *data) {
 	struct sway_cursor *cursor = wl_container_of(listener, cursor, button);
 	struct wlr_event_pointer_button *event = data;
-	cursor_handle_activity(cursor, event->device);
 
 	if (event->state == WLR_BUTTON_PRESSED) {
 		cursor->pressed_button_count++;
@@ -390,6 +389,7 @@ static void handle_pointer_button(struct wl_listener *listener, void *data) {
 		}
 	}
 
+	cursor_handle_activity(cursor, event->device);
 	dispatch_cursor_button(cursor, event->device,
 			event->time_msec, event->button, event->state);
 	transaction_commit_dirty();