Refactor seat_get_focus functions

Fixes #2467.

This commit introduces seat_get_focus_inactive_floating to supplement
seat_get_focus_inactive_tiling, and uses it during `focus mode_toggle`
which fixes a focus bug.

This also refactors the seat_get_focus_inactive functions so that they
do their selection logic themselves rather than offloading it to
seat_get_focus_by_type which was getting bloated. seat_get_focus_by_type
is now removed.

Lastly, this commit changes seat_get_focus to just return the first
container in the focus stack rather than looping and calling
seat_get_focus_by_type.

include/sway/input/seat.h

@@ -166,6 +166,9 @@ void seat_begin_resize_floating(struct sway_seat *seat,
 void seat_begin_resize_tiling(struct sway_seat *seat,
 		struct sway_container *con, uint32_t button, enum wlr_edges edge);
 
+struct sway_container *seat_get_focus_inactive_floating(struct sway_seat *seat,
+		struct sway_container *container);
+
 void seat_end_mouse_operation(struct sway_seat *seat);
 
 void seat_pointer_notify_button(struct sway_seat *seat, uint32_t time_msec,

sway/commands/focus.c

@@ -46,14 +46,13 @@ static struct cmd_results *focus_mode(struct sway_container *con,
 
 	struct sway_container *new_focus = NULL;
 	if (floating) {
-		new_focus = seat_get_focus_inactive(seat, ws->sway_workspace->floating);
+		new_focus = seat_get_focus_inactive_floating(seat, ws);
 	} else {
 		new_focus = seat_get_focus_inactive_tiling(seat, ws);
 	}
-	if (!new_focus) {
+	if (new_focus) {
-		new_focus = ws;
+		seat_set_focus(seat, new_focus);
 	}
-	seat_set_focus(seat, new_focus);
 	return cmd_results_new(CMD_SUCCESS, NULL, NULL);
 }
 

sway/input/seat.c

@@ -124,42 +124,6 @@ static void seat_send_focus(struct sway_container *con,
 	}
 }
 
-static struct sway_container *seat_get_focus_by_type(struct sway_seat *seat,
-		struct sway_container *container, enum sway_container_type type,
-		bool only_tiling) {
-	if (container->type == C_VIEW) {
-		return container;
-	}
-
-	struct sway_container *floating =
-		container->type == C_WORKSPACE && !only_tiling ?
-		container->sway_workspace->floating : NULL;
-	if (container->children->length == 0 &&
-			(!floating || floating->children->length == 0)) {
-		return container;
-	}
-
-	struct sway_seat_container *current = NULL;
-	wl_list_for_each(current, &seat->focus_stack, link) {
-		if (current->container->type != type && type != C_TYPES) {
-			continue;
-		}
-
-		if (container_has_ancestor(current->container, container)) {
-			if (only_tiling &&
-					container_is_floating_or_child(current->container)) {
-				continue;
-			}
-			return current->container;
-		}
-		if (floating && container_has_ancestor(current->container, floating)) {
-			return current->container;
-		}
-	}
-
-	return NULL;
-}
-
 void seat_focus_inactive_children_for_each(struct sway_seat *seat,
 		struct sway_container *container,
 		void (*f)(struct sway_container *container, void *data), void *data) {
@@ -175,8 +139,18 @@ void seat_focus_inactive_children_for_each(struct sway_seat *seat,
 }
 
 struct sway_container *seat_get_focus_inactive_view(struct sway_seat *seat,
-		struct sway_container *container) {
+		struct sway_container *ancestor) {
-	return seat_get_focus_by_type(seat, container, C_VIEW, false);
+	if (ancestor->type == C_VIEW) {
+		return ancestor;
+	}
+	struct sway_seat_container *current;
+	wl_list_for_each(current, &seat->focus_stack, link) {
+		struct sway_container *con = current->container;
+		if (con->type == C_VIEW && container_has_ancestor(con, ancestor)) {
+			return con;
+		}
+	}
+	return NULL;
 }
 
 static void handle_seat_container_destroy(struct wl_listener *listener,
@@ -198,7 +172,7 @@ static void handle_seat_container_destroy(struct wl_listener *listener,
 	if (set_focus) {
 		struct sway_container *next_focus = NULL;
 		while (next_focus == NULL) {
-			next_focus = seat_get_focus_by_type(seat, parent, C_VIEW, false);
+			next_focus = seat_get_focus_inactive_view(seat, parent);
 
 			if (next_focus == NULL && parent->type == C_WORKSPACE) {
 				next_focus = parent;
@@ -653,8 +627,7 @@ void seat_set_focus_warp(struct sway_seat *seat,
 	// find new output's old workspace, which might have to be removed if empty
 	struct sway_container *new_output_last_ws = NULL;
 	if (last_output && new_output && last_output != new_output) {
-		new_output_last_ws =
+		new_output_last_ws = seat_get_active_child(seat, new_output);
-			seat_get_focus_by_type(seat, new_output, C_WORKSPACE, false);
 	}
 
 	if (container && container->parent) {
@@ -877,22 +850,71 @@ void seat_set_exclusive_client(struct sway_seat *seat,
 }
 
 struct sway_container *seat_get_focus_inactive(struct sway_seat *seat,
-		struct sway_container *container) {
+		struct sway_container *con) {
-	return seat_get_focus_by_type(seat, container, C_TYPES, false);
+	if (con->type == C_WORKSPACE && !con->children->length &&
+			!con->sway_workspace->floating->children->length) {
+		return con;
+	}
+	if (con->type == C_VIEW) {
+		return con;
+	}
+	struct sway_seat_container *current;
+	wl_list_for_each(current, &seat->focus_stack, link) {
+		if (container_has_ancestor(current->container, con)) {
+			return current->container;
+		}
+	}
+	return NULL;
 }
 
 struct sway_container *seat_get_focus_inactive_tiling(struct sway_seat *seat,
-		struct sway_container *container) {
+		struct sway_container *ancestor) {
-	return seat_get_focus_by_type(seat, container, C_TYPES, true);
+	if (ancestor->type == C_WORKSPACE && !ancestor->children->length) {
+		return ancestor;
+	}
+	struct sway_seat_container *current;
+	wl_list_for_each(current, &seat->focus_stack, link) {
+		struct sway_container *con = current->container;
+		if (con->layout != L_FLOATING && !container_is_floating_or_child(con) &&
+				container_has_ancestor(current->container, ancestor)) {
+			return con;
+		}
+	}
+	return NULL;
+}
+
+struct sway_container *seat_get_focus_inactive_floating(struct sway_seat *seat,
+		struct sway_container *ancestor) {
+	if (ancestor->type == C_WORKSPACE &&
+			!ancestor->sway_workspace->floating->children->length) {
+		return NULL;
+	}
+	struct sway_seat_container *current;
+	wl_list_for_each(current, &seat->focus_stack, link) {
+		struct sway_container *con = current->container;
+		if (con->layout != L_FLOATING && container_is_floating_or_child(con) &&
+				container_has_ancestor(current->container, ancestor)) {
+			return con;
+		}
+	}
+	return NULL;
+}
+
+static bool impl_focus_active_child(struct sway_container *con, void *data) {
+	struct sway_container *parent = data;
+	return con->parent == parent && con->layout != L_FLOATING;
 }
 
 struct sway_container *seat_get_active_child(struct sway_seat *seat,
-		struct sway_container *container) {
+		struct sway_container *parent) {
-	struct sway_seat_container *current = NULL;
+	if (parent->type == C_VIEW) {
+		return parent;
+	}
+	struct sway_seat_container *current;
 	wl_list_for_each(current, &seat->focus_stack, link) {
-		if (current->container->parent == container &&
+		struct sway_container *con = current->container;
-				current->container->layout != L_FLOATING) {
+		if (con->parent == parent && con->layout != L_FLOATING) {
-			return current->container;
+			return con;
 		}
 	}
 	return NULL;
@@ -902,7 +924,9 @@ struct sway_container *seat_get_focus(struct sway_seat *seat) {
 	if (!seat->has_focus) {
 		return NULL;
 	}
-	return seat_get_focus_inactive(seat, &root_container);
+	struct sway_seat_container *current =
+		wl_container_of(seat->focus_stack.next, current, link);
+	return current->container;
 }
 
 void seat_apply_config(struct sway_seat *seat,