diff --git a/sway/container.c b/sway/container.c
index 41d21c3b..67a34551 100644
--- a/sway/container.c
+++ b/sway/container.c
@@ -86,6 +86,7 @@ swayc_t *new_output(wlc_handle handle) {
 	
 	// Find position for it
 	if (oc && oc->x != -1 && oc->y != -1) {
+		sway_log(L_DEBUG, "Set %s position to %d, %d", name, oc->x, oc->y);
 		output->x = oc->x;
 		output->y = oc->y;
 	} else {
diff --git a/sway/layout.c b/sway/layout.c
index 7f3adc31..7a7ccc0e 100644
--- a/sway/layout.c
+++ b/sway/layout.c
@@ -169,8 +169,6 @@ void arrange_windows(swayc_t *container, double width, double height) {
 		for (i = 0; i < container->children->length; ++i) {
 			swayc_t *child = container->children->items[i];
 			sway_log(L_DEBUG, "Arranging output at %d", x);
-			child->x = x;
-			child->y = y;
 			arrange_windows(child, -1, -1);
 			x += child->width;
 		}
@@ -340,19 +338,79 @@ swayc_t *get_swayc_in_direction(swayc_t *container, enum movement_direction dir)
 		// Test if we can even make a difference here
 		bool can_move = false;
 		int diff = 0;
-		if (dir == MOVE_LEFT || dir == MOVE_RIGHT) {
-			if (parent->layout == L_HORIZ || parent->type == C_ROOT) {
+		int i;
+		if (parent->type == C_ROOT) {
+			// Find the next output
+			int target = -1, max_x = 0, max_y = 0, self = -1;
+			sway_log(L_DEBUG, "Moving between outputs");
+			
+			for (i = 0; i < parent->children->length; ++i) {
+				swayc_t *next = parent->children->items[i];
+				if (next == container) {
+					self = i;
+					sway_log(L_DEBUG, "self is %p %d", next, self);
+					continue;
+				}
+				if (next->type == C_OUTPUT) {
+					sway_log(L_DEBUG, "Testing with %p %d (dir %d)", next, i, dir);
+					// Check if it's more extreme
+					if (dir == MOVE_RIGHT) {
+						if (container->x + container->width <= next->x) {
+							if (target == -1 || next->x < max_x) {
+								target = i;
+								max_x = next->x;
+							}
+						}
+					} else if (dir == MOVE_LEFT) {
+						sway_log(L_DEBUG, "next->x: %d, next->width: %d, container->x: %d",
+								(int)next->x, (int)next->width, (int)container->x);
+						if (container->x > next->x + next->width) {
+							sway_log(L_DEBUG, "match");
+							if (target == -1 || max_x < next->x) {
+								sway_log(L_DEBUG, "hit");
+								target = i;
+								max_x = next->x;
+							}
+						}
+					} else if (dir == MOVE_DOWN) {
+						if (container->y + container->height <= next->y) {
+							if (target == -1 || next->y < max_y) {
+								target = i;
+								max_y = next->y;
+							}
+						}
+					} else if (dir == MOVE_UP) {
+						if (container->y > next->y + next->height) {
+							if (target == -1 || max_y < next->y) {
+								target = i;
+								max_y = next->y;
+							}
+						}
+					}
+				}
+			}
+
+			if (target == -1) {
+				can_move = false;
+			} else {
 				can_move = true;
-				diff = dir == MOVE_LEFT ? -1 : 1;
+				diff = target - self;
 			}
 		} else {
-			if (parent->layout == L_VERT) {
-				can_move = true;
-				diff = dir == MOVE_UP ? -1 : 1;
+			if (dir == MOVE_LEFT || dir == MOVE_RIGHT) {
+				if (parent->layout == L_HORIZ) {
+					can_move = true;
+					diff = dir == MOVE_LEFT ? -1 : 1;
+				}
+			} else {
+				if (parent->layout == L_VERT) {
+					can_move = true;
+					diff = dir == MOVE_UP ? -1 : 1;
+				}
 			}
 		}
+
 		if (can_move) {
-			int i;
 			for (i = 0; i < parent->children->length; ++i) {
 				swayc_t *child = parent->children->items[i];
 				if (child == container) {