Initialize download index disk cache (#9179)

app/src/main/java/eu/kanade/tachiyomi/data/download/DownloadCache.kt

@@ -1,6 +1,8 @@
 package eu.kanade.tachiyomi.data.download
 
+import android.app.Application
 import android.content.Context
+import android.net.Uri
 import androidx.core.net.toUri
 import com.hippo.unifile.UniFile
 import eu.kanade.core.util.mapNotNullKeys
@@ -14,6 +16,7 @@ import kotlinx.coroutines.async
 import kotlinx.coroutines.awaitAll
 import kotlinx.coroutines.channels.Channel
 import kotlinx.coroutines.delay
+import kotlinx.coroutines.ensureActive
 import kotlinx.coroutines.flow.MutableStateFlow
 import kotlinx.coroutines.flow.SharingStarted
 import kotlinx.coroutines.flow.debounce
@@ -23,7 +26,20 @@ import kotlinx.coroutines.flow.onStart
 import kotlinx.coroutines.flow.receiveAsFlow
 import kotlinx.coroutines.flow.shareIn
 import kotlinx.coroutines.flow.stateIn
+import kotlinx.coroutines.runBlocking
+import kotlinx.coroutines.sync.Mutex
+import kotlinx.coroutines.sync.withLock
 import kotlinx.coroutines.withTimeoutOrNull
+import kotlinx.serialization.KSerializer
+import kotlinx.serialization.Serializable
+import kotlinx.serialization.decodeFromByteArray
+import kotlinx.serialization.descriptors.PrimitiveKind
+import kotlinx.serialization.descriptors.PrimitiveSerialDescriptor
+import kotlinx.serialization.descriptors.SerialDescriptor
+import kotlinx.serialization.encodeToByteArray
+import kotlinx.serialization.encoding.Decoder
+import kotlinx.serialization.encoding.Encoder
+import kotlinx.serialization.protobuf.ProtoBuf
 import logcat.LogPriority
 import tachiyomi.core.util.lang.launchIO
 import tachiyomi.core.util.lang.launchNonCancellable
@@ -34,7 +50,7 @@ import tachiyomi.domain.manga.model.Manga
 import tachiyomi.domain.source.service.SourceManager
 import uy.kohesive.injekt.Injekt
 import uy.kohesive.injekt.api.get
-import java.util.concurrent.ConcurrentHashMap
+import java.io.File
 import kotlin.time.Duration.Companion.hours
 import kotlin.time.Duration.Companion.seconds
 
@@ -76,7 +92,11 @@ class DownloadCache(
         .debounce(1000L) // Don't notify if it finishes quickly enough
         .stateIn(scope, SharingStarted.WhileSubscribed(), false)
 
-    private var rootDownloadsDir = RootDirectory(getDirectoryFromPreference())
+    private val diskCacheFile: File
+        get() = File(context.cacheDir, "dl_index_cache")
+
+    private val rootDownloadsDirLock = Mutex()
+    private var rootDownloadsDir: RootDirectory
 
     init {
         downloadPreferences.downloadsDirectory().changes()
@@ -85,6 +105,21 @@ class DownloadCache(
                 invalidateCache()
             }
             .launchIn(scope)
+
+        rootDownloadsDir = runBlocking(Dispatchers.IO) {
+            try {
+                val diskCache = diskCacheFile.inputStream().use {
+                    ProtoBuf.decodeFromByteArray<RootDirectory>(it.readBytes())
+                }
+                lastRenew = 1 // Just so that the banner won't show up
+                diskCache
+            } catch (e: Throwable) {
+                diskCacheFile.delete()
+                null
+            }
+        } ?: RootDirectory(getDirectoryFromPreference())
+
+        notifyChanges()
     }
 
     /**
@@ -158,27 +193,28 @@ class DownloadCache(
      * @param mangaUniFile the directory of the manga.
      * @param manga the manga of the chapter.
      */
-    @Synchronized
-    fun addChapter(chapterDirName: String, mangaUniFile: UniFile, manga: Manga) {
-        // Retrieve the cached source directory or cache a new one
-        var sourceDir = rootDownloadsDir.sourceDirs[manga.source]
-        if (sourceDir == null) {
-            val source = sourceManager.get(manga.source) ?: return
-            val sourceUniFile = provider.findSourceDir(source) ?: return
-            sourceDir = SourceDirectory(sourceUniFile)
-            rootDownloadsDir.sourceDirs += manga.source to sourceDir
-        }
+    suspend fun addChapter(chapterDirName: String, mangaUniFile: UniFile, manga: Manga) {
+        rootDownloadsDirLock.withLock {
+            // Retrieve the cached source directory or cache a new one
+            var sourceDir = rootDownloadsDir.sourceDirs[manga.source]
+            if (sourceDir == null) {
+                val source = sourceManager.get(manga.source) ?: return
+                val sourceUniFile = provider.findSourceDir(source) ?: return
+                sourceDir = SourceDirectory(sourceUniFile)
+                rootDownloadsDir.sourceDirs += manga.source to sourceDir
+            }
 
-        // Retrieve the cached manga directory or cache a new one
-        val mangaDirName = provider.getMangaDirName(manga.title)
-        var mangaDir = sourceDir.mangaDirs[mangaDirName]
-        if (mangaDir == null) {
-            mangaDir = MangaDirectory(mangaUniFile)
-            sourceDir.mangaDirs += mangaDirName to mangaDir
-        }
+            // Retrieve the cached manga directory or cache a new one
+            val mangaDirName = provider.getMangaDirName(manga.title)
+            var mangaDir = sourceDir.mangaDirs[mangaDirName]
+            if (mangaDir == null) {
+                mangaDir = MangaDirectory(mangaUniFile)
+                sourceDir.mangaDirs += mangaDirName to mangaDir
+            }
 
-        // Save the chapter directory
-        mangaDir.chapterDirs += chapterDirName
+            // Save the chapter directory
+            mangaDir.chapterDirs += chapterDirName
+        }
 
         notifyChanges()
     }
@@ -189,13 +225,14 @@ class DownloadCache(
      * @param chapter the chapter to remove.
      * @param manga the manga of the chapter.
      */
-    @Synchronized
-    fun removeChapter(chapter: Chapter, manga: Manga) {
-        val sourceDir = rootDownloadsDir.sourceDirs[manga.source] ?: return
-        val mangaDir = sourceDir.mangaDirs[provider.getMangaDirName(manga.title)] ?: return
-        provider.getValidChapterDirNames(chapter.name, chapter.scanlator).forEach {
-            if (it in mangaDir.chapterDirs) {
-                mangaDir.chapterDirs -= it
+    suspend fun removeChapter(chapter: Chapter, manga: Manga) {
+        rootDownloadsDirLock.withLock {
+            val sourceDir = rootDownloadsDir.sourceDirs[manga.source] ?: return
+            val mangaDir = sourceDir.mangaDirs[provider.getMangaDirName(manga.title)] ?: return
+            provider.getValidChapterDirNames(chapter.name, chapter.scanlator).forEach {
+                if (it in mangaDir.chapterDirs) {
+                    mangaDir.chapterDirs -= it
+                }
             }
         }
 
@@ -208,14 +245,15 @@ class DownloadCache(
      * @param chapters the list of chapter to remove.
      * @param manga the manga of the chapter.
      */
-    @Synchronized
-    fun removeChapters(chapters: List<Chapter>, manga: Manga) {
-        val sourceDir = rootDownloadsDir.sourceDirs[manga.source] ?: return
-        val mangaDir = sourceDir.mangaDirs[provider.getMangaDirName(manga.title)] ?: return
-        chapters.forEach { chapter ->
-            provider.getValidChapterDirNames(chapter.name, chapter.scanlator).forEach {
-                if (it in mangaDir.chapterDirs) {
-                    mangaDir.chapterDirs -= it
+    suspend fun removeChapters(chapters: List<Chapter>, manga: Manga) {
+        rootDownloadsDirLock.withLock {
+            val sourceDir = rootDownloadsDir.sourceDirs[manga.source] ?: return
+            val mangaDir = sourceDir.mangaDirs[provider.getMangaDirName(manga.title)] ?: return
+            chapters.forEach { chapter ->
+                provider.getValidChapterDirNames(chapter.name, chapter.scanlator).forEach {
+                    if (it in mangaDir.chapterDirs) {
+                        mangaDir.chapterDirs -= it
+                    }
                 }
             }
         }
@@ -228,20 +266,22 @@ class DownloadCache(
      *
      * @param manga the manga to remove.
      */
-    @Synchronized
-    fun removeManga(manga: Manga) {
-        val sourceDir = rootDownloadsDir.sourceDirs[manga.source] ?: return
-        val mangaDirName = provider.getMangaDirName(manga.title)
-        if (sourceDir.mangaDirs.containsKey(mangaDirName)) {
-            sourceDir.mangaDirs -= mangaDirName
+    suspend fun removeManga(manga: Manga) {
+        rootDownloadsDirLock.withLock {
+            val sourceDir = rootDownloadsDir.sourceDirs[manga.source] ?: return
+            val mangaDirName = provider.getMangaDirName(manga.title)
+            if (sourceDir.mangaDirs.containsKey(mangaDirName)) {
+                sourceDir.mangaDirs -= mangaDirName
+            }
         }
 
         notifyChanges()
     }
 
-    @Synchronized
-    fun removeSource(source: Source) {
-        rootDownloadsDir.sourceDirs -= source.id
+    suspend fun removeSource(source: Source) {
+        rootDownloadsDirLock.withLock {
+            rootDownloadsDir.sourceDirs -= source.id
+        }
 
         notifyChanges()
     }
@@ -287,46 +327,48 @@ class DownloadCache(
                 }
             }
 
-            val sourceDirs = rootDownloadsDir.dir.listFiles().orEmpty()
-                .associate { it.name to SourceDirectory(it) }
-                .mapNotNullKeys { entry ->
-                    sources.find {
-                        provider.getSourceDirName(it).equals(entry.key, ignoreCase = true)
-                    }?.id
-                }
+            rootDownloadsDirLock.withLock {
+                val sourceDirs = rootDownloadsDir.dir.listFiles().orEmpty()
+                    .associate { it.name to SourceDirectory(it) }
+                    .mapNotNullKeys { entry ->
+                        sources.find {
+                            provider.getSourceDirName(it).equals(entry.key, ignoreCase = true)
+                        }?.id
+                    }
 
-            rootDownloadsDir.sourceDirs = sourceDirs
-
-            sourceDirs.values
-                .map { sourceDir ->
-                    async {
-                        val mangaDirs = sourceDir.dir.listFiles().orEmpty()
-                            .filterNot { it.name.isNullOrBlank() }
-                            .associate { it.name!! to MangaDirectory(it) }
-
-                        sourceDir.mangaDirs = ConcurrentHashMap(mangaDirs)
-
-                        mangaDirs.values.forEach { mangaDir ->
-                            val chapterDirs = mangaDir.dir.listFiles().orEmpty()
-                                .mapNotNull {
-                                    when {
-                                        // Ignore incomplete downloads
-                                        it.name?.endsWith(Downloader.TMP_DIR_SUFFIX) == true -> null
-                                        // Folder of images
-                                        it.isDirectory -> it.name
-                                        // CBZ files
-                                        it.isFile && it.name?.endsWith(".cbz") == true -> it.name!!.substringBeforeLast(".cbz")
-                                        // Anything else is irrelevant
-                                        else -> null
+                rootDownloadsDir.sourceDirs = sourceDirs
+
+                sourceDirs.values
+                    .map { sourceDir ->
+                        async {
+                            sourceDir.mangaDirs = sourceDir.dir.listFiles().orEmpty()
+                                .filterNot { it.name.isNullOrBlank() }
+                                .associate { it.name!! to MangaDirectory(it) }
+
+                            sourceDir.mangaDirs.values.forEach { mangaDir ->
+                                val chapterDirs = mangaDir.dir.listFiles().orEmpty()
+                                    .mapNotNull {
+                                        when {
+                                            // Ignore incomplete downloads
+                                            it.name?.endsWith(Downloader.TMP_DIR_SUFFIX) == true -> null
+                                            // Folder of images
+                                            it.isDirectory -> it.name
+                                            // CBZ files
+                                            it.isFile && it.name?.endsWith(".cbz") == true -> it.name!!.substringBeforeLast(
+                                                ".cbz",
+                                            )
+                                            // Anything else is irrelevant
+                                            else -> null
+                                        }
                                     }
-                                }
-                                .toMutableSet()
+                                    .toMutableSet()
 
-                            mangaDir.chapterDirs = chapterDirs
+                                mangaDir.chapterDirs = chapterDirs
+                            }
                         }
                     }
-                }
-                .awaitAll()
+                    .awaitAll()
+            }
 
             _isInitializing.emit(false)
         }.also {
@@ -335,6 +377,7 @@ class DownloadCache(
                     logcat(LogPriority.ERROR, exception) { "Failed to create download cache" }
                 }
                 lastRenew = System.currentTimeMillis()
+
                 notifyChanges()
             }
         }
@@ -351,29 +394,67 @@ class DownloadCache(
         scope.launchNonCancellable {
             _changes.send(Unit)
         }
+        updateDiskCache()
+    }
+
+    private var updateDiskCacheJob: Job? = null
+    private fun updateDiskCache() {
+        updateDiskCacheJob?.cancel()
+        updateDiskCacheJob = scope.launchIO {
+            delay(1000)
+            ensureActive()
+            val bytes = ProtoBuf.encodeToByteArray(rootDownloadsDir)
+            ensureActive()
+            try {
+                diskCacheFile.writeBytes(bytes)
+            } catch (e: Throwable) {
+                logcat(
+                    priority = LogPriority.ERROR,
+                    throwable = e,
+                    message = { "Failed to write disk cache file" },
+                )
+            }
+        }
     }
 }
 
 /**
  * Class to store the files under the root downloads directory.
  */
+@Serializable
 private class RootDirectory(
+    @Serializable(with = UniFileAsStringSerializer::class)
     val dir: UniFile,
-    var sourceDirs: ConcurrentHashMap<Long, SourceDirectory> = ConcurrentHashMap(),
+    var sourceDirs: Map<Long, SourceDirectory> = mapOf(),
 )
 
 /**
  * Class to store the files under a source directory.
  */
+@Serializable
 private class SourceDirectory(
+    @Serializable(with = UniFileAsStringSerializer::class)
     val dir: UniFile,
-    var mangaDirs: ConcurrentHashMap<String, MangaDirectory> = ConcurrentHashMap(),
+    var mangaDirs: Map<String, MangaDirectory> = mapOf(),
 )
 
 /**
  * Class to store the files under a manga directory.
  */
+@Serializable
 private class MangaDirectory(
+    @Serializable(with = UniFileAsStringSerializer::class)
     val dir: UniFile,
     var chapterDirs: MutableSet<String> = mutableSetOf(),
 )
+
+private object UniFileAsStringSerializer : KSerializer<UniFile> {
+    override val descriptor: SerialDescriptor = PrimitiveSerialDescriptor("UniFile", PrimitiveKind.STRING)
+
+    override fun serialize(encoder: Encoder, value: UniFile) {
+        return encoder.encodeString(value.uri.toString())
+    }
+    override fun deserialize(decoder: Decoder): UniFile {
+        return UniFile.fromUri(Injekt.get<Application>(), Uri.parse(decoder.decodeString()))
+    }
+}

app/src/main/java/eu/kanade/tachiyomi/data/download/DownloadManager.kt

@@ -325,7 +325,7 @@ class DownloadManager(
      * @param oldChapter the existing chapter with the old name.
      * @param newChapter the target chapter with the new name.
      */
-    fun renameChapter(source: Source, manga: Manga, oldChapter: Chapter, newChapter: Chapter) {
+    suspend fun renameChapter(source: Source, manga: Manga, oldChapter: Chapter, newChapter: Chapter) {
         val oldNames = provider.getValidChapterDirNames(oldChapter.name, oldChapter.scanlator)
         val mangaDir = provider.getMangaDir(manga.title, source)
 

app/src/main/java/eu/kanade/tachiyomi/data/download/Downloader.kt

@@ -527,7 +527,7 @@ class Downloader(
      * @param tmpDir the directory where the download is currently stored.
      * @param dirname the real (non temporary) directory name of the download.
      */
-    private fun ensureSuccessfulDownload(
+    private suspend fun ensureSuccessfulDownload(
         download: Download,
         mangaDir: UniFile,
         tmpDir: UniFile,