Fix weird behaviour in library when switching display mode (#5640)

app/src/main/java/eu/kanade/tachiyomi/ui/library/LibraryController.kt

@@ -196,7 +196,12 @@ class LibraryController(
             when (group) {
                 is LibrarySettingsSheet.Filter.FilterGroup -> onFilterChanged()
                 is LibrarySettingsSheet.Sort.SortGroup -> onSortChanged()
-                is LibrarySettingsSheet.Display.DisplayGroup -> reattachAdapter()
+                is LibrarySettingsSheet.Display.DisplayGroup -> {
+                    if (!preferences.categorisedDisplaySettings().get() || activeCategory == 0) {
+                        // Reattach adapter when flow preference change
+                        reattachAdapter()
+                    }
+                }
                 is LibrarySettingsSheet.Display.BadgeGroup -> onBadgeSettingChanged()
                 is LibrarySettingsSheet.Display.TabsGroup -> onTabsSettingsChanged()
             }