diff --git a/src/gs-shell.c b/src/gs-shell.c
index ec15c3f132ed744fd0f61afe4318aaa331aaa1d4..214a201c3875a4c60a96183436e0cf2feedf8d95 100644
--- a/src/gs-shell.c
+++ b/src/gs-shell.c
@@ -1000,6 +1000,17 @@ search_bar_search_mode_enabled_changed_cb (GtkSearchBar *search_bar,
 		gs_shell_go_back (shell);
 }
 
+static void
+go_back (GsShell *shell)
+{
+	if (adw_leaflet_get_adjacent_child (shell->details_leaflet,
+					    ADW_NAVIGATION_DIRECTION_BACK)) {
+		gtk_widget_activate (shell->button_back2);
+	} else {
+		gtk_widget_activate (shell->button_back);
+	}
+}
+
 static gboolean
 window_key_pressed_cb (GtkEventControllerKey *key_controller,
                        guint                  keyval,
@@ -1008,13 +1019,12 @@ window_key_pressed_cb (GtkEventControllerKey *key_controller,
                        GsShell               *shell)
 {
 	gboolean is_rtl = gtk_widget_get_direction (shell->button_back) == GTK_TEXT_DIR_RTL;
+	gboolean is_alt = (state & (GDK_SHIFT_MASK | GDK_CONTROL_MASK | GDK_ALT_MASK)) == GDK_ALT_MASK;
 
-	if ((!is_rtl && state == GDK_ALT_MASK && keyval == GDK_KEY_Left) ||
-	    (is_rtl && state == GDK_ALT_MASK && keyval == GDK_KEY_Right) ||
+	if ((!is_rtl && is_alt && keyval == GDK_KEY_Left) ||
+	    (is_rtl && is_alt && keyval == GDK_KEY_Right) ||
 	    keyval == GDK_KEY_Back) {
-		/* GTK will only actually activate the one which is visible */
-		gtk_widget_activate (shell->button_back);
-		gtk_widget_activate (shell->button_back2);
+		go_back (shell);
 		return GDK_EVENT_STOP;
 	}
 
@@ -1028,9 +1038,7 @@ window_button_pressed_cb (GtkGestureClick *click_gesture,
                           gdouble          y,
                           GsShell         *shell)
 {
-	/* GTK will only actually activate the one which is visible */
-	gtk_widget_activate (shell->button_back);
-	gtk_widget_activate (shell->button_back2);
+	go_back (shell);
 
 	gtk_gesture_set_state (GTK_GESTURE (click_gesture), GTK_EVENT_SEQUENCE_CLAIMED);
 }