diff --git a/include/swaybar/bar.h b/include/swaybar/bar.h
index 031993b5..84619237 100644
--- a/include/swaybar/bar.h
+++ b/include/swaybar/bar.h
@@ -23,6 +23,7 @@ struct swaybar {
 	// only relevant when bar is in "hide" mode
 	bool visible_by_modifier;
 	bool visible_by_urgency;
+	bool visible_by_mode;
 	bool visible;
 
 	struct wl_display *display;
diff --git a/swaybar/bar.c b/swaybar/bar.c
index ca7cd88c..a95464f5 100644
--- a/swaybar/bar.c
+++ b/swaybar/bar.c
@@ -148,7 +148,8 @@ bool determine_bar_visibility(struct swaybar *bar, bool moving_layer) {
 	struct swaybar_config *config = bar->config;
 	bool visible = !(strcmp(config->mode, "invisible") == 0 ||
 		(strcmp(config->mode, config->hidden_state) == 0 // both "hide"
-			&& !bar->visible_by_modifier && !bar->visible_by_urgency));
+			&& !bar->visible_by_modifier && !bar->visible_by_urgency
+			&& !bar->visible_by_mode));
 
 	// Create/destroy layer surfaces as needed
 	struct swaybar_output *output;
diff --git a/swaybar/ipc.c b/swaybar/ipc.c
index 1cc55b43..a096f01a 100644
--- a/swaybar/ipc.c
+++ b/swaybar/ipc.c
@@ -499,6 +499,13 @@ static bool handle_bar_state_update(struct swaybar *bar, json_object *event) {
 	json_object *visible_by_modifier;
 	json_object_object_get_ex(event, "visible_by_modifier", &visible_by_modifier);
 	bar->visible_by_modifier = json_object_get_boolean(visible_by_modifier);
+	if (bar->visible_by_modifier) {
+		// If the bar is visible by modifier, clear both visible by mode and
+		// urgency as modifier has precedence and the bar should be hidden
+		// again when it is no longer visible by modifier.
+		bar->visible_by_mode = false;
+		bar->visible_by_urgency = false;
+	}
 	return determine_bar_visibility(bar, false);
 }
 
@@ -578,6 +585,8 @@ bool handle_ipc_readable(struct swaybar *bar) {
 			const char *change = json_object_get_string(json_change);
 			free(bar->mode);
 			bar->mode = strcmp(change, "default") != 0 ? strdup(change) : NULL;
+			bar->visible_by_mode = bar->mode != NULL;
+			determine_bar_visibility(bar, false);
 		} else {
 			sway_log(SWAY_ERROR, "failed to parse response");
 			bar_is_dirty = false;