Revert "Prevent view from being removed if a to is found (#5135) (#5152)"

This reverts commit 974275a42993534b6d63ae6ba34b75c3c21b7627.

app/src/main/java/eu/kanade/tachiyomi/ui/base/controller/OneWayFadeChangeHandler.kt

@@ -31,12 +31,11 @@ class OneWayFadeChangeHandler : FadeChangeHandler {
         if (to != null) {
             val start: Float = if (toAddedToContainer) 0F else to.alpha
             animator.play(ObjectAnimator.ofFloat(to, View.ALPHA, start, 1f))
-        } else {
-            if (from != null && (!isPush || removesFromViewOnPush())) {
-                container.removeView(from)
-            }
         }
 
+        if (from != null && (!isPush || removesFromViewOnPush())) {
+            container.removeView(from)
+        }
         return animator
     }