Rewrite sources. Implement Batoto and Kissmanga

app/build.gradle

@@ -131,6 +131,9 @@ dependencies {
     // JSON
     compile 'com.google.code.gson:gson:2.6.2'
 
+    // YAML
+    compile 'org.yaml:snakeyaml:1.17'
+
     // JavaScript engine
     compile 'com.squareup.duktape:duktape-android:0.9.5'
 

app/src/main/java/eu/kanade/tachiyomi/data/download/DownloadManager.kt

@@ -13,6 +13,7 @@ import eu.kanade.tachiyomi.data.download.model.DownloadQueue
 import eu.kanade.tachiyomi.data.preference.PreferencesHelper
 import eu.kanade.tachiyomi.data.preference.getOrDefault
 import eu.kanade.tachiyomi.data.source.SourceManager
+import eu.kanade.tachiyomi.data.source.base.OnlineSource
 import eu.kanade.tachiyomi.data.source.base.Source
 import eu.kanade.tachiyomi.data.source.model.Page
 import eu.kanade.tachiyomi.util.DiskUtils
@@ -108,7 +109,7 @@ class DownloadManager(private val context: Context, private val sourceManager: S
 
     // Create a download object for every chapter and add them to the downloads queue
     fun downloadChapters(manga: Manga, chapters: List<Chapter>) {
-        val source = sourceManager.get(manga.source)
+        val source = sourceManager.get(manga.source) as? OnlineSource ?: return
 
         // Used to avoid downloading chapters with the same name
         val addedChapters = ArrayList<String>()
@@ -182,8 +183,8 @@ class DownloadManager(private val context: Context, private val sourceManager: S
         DiskUtils.createDirectory(download.directory)
 
         val pageListObservable = if (download.pages == null)
-        // Pull page list from network and add them to download object
-            download.source.pullPageListFromNetwork(download.chapter.url)
+            // Pull page list from network and add them to download object
+            download.source.fetchPageListFromNetwork(download.chapter)
                     .doOnNext { pages ->
                         download.pages = pages
                         savePageList(download)
@@ -199,7 +200,7 @@ class DownloadManager(private val context: Context, private val sourceManager: S
                         download.status = Download.DOWNLOADING
                     }
                     // Get all the URLs to the source images, fetch pages if necessary
-                    .flatMap { download.source.getAllImageUrlsFromPageList(it) }
+                    .flatMap { download.source.fetchAllImageUrlsFromPageList(it) }
                     // Start downloading images, consider we can have downloaded images already
                     .concatMap { page -> getOrDownloadImage(page, download) }
                     // Do when page is downloaded.
@@ -251,9 +252,9 @@ class DownloadManager(private val context: Context, private val sourceManager: S
     }
 
     // Save image on disk
-    private fun downloadImage(page: Page, source: Source, directory: File, filename: String): Observable<Page> {
+    private fun downloadImage(page: Page, source: OnlineSource, directory: File, filename: String): Observable<Page> {
         page.status = Page.DOWNLOAD_IMAGE
-        return source.getImageProgressResponse(page)
+        return source.imageResponse(page)
                 .flatMap {
                     try {
                         val file = File(directory, filename)
@@ -376,7 +377,7 @@ class DownloadManager(private val context: Context, private val sourceManager: S
     }
 
     fun getAbsoluteMangaDirectory(source: Source, manga: Manga): File {
-        val mangaRelativePath = source.visibleName +
+        val mangaRelativePath = source.toString() +
                 File.separator +
                 manga.title.replace("[^\\sa-zA-Z0-9.-]".toRegex(), "_")
 

app/src/main/java/eu/kanade/tachiyomi/data/download/model/Download.java

@@ -5,12 +5,12 @@ import java.util.List;
 
 import eu.kanade.tachiyomi.data.database.models.Chapter;
 import eu.kanade.tachiyomi.data.database.models.Manga;
-import eu.kanade.tachiyomi.data.source.base.Source;
+import eu.kanade.tachiyomi.data.source.base.OnlineSource;
 import eu.kanade.tachiyomi.data.source.model.Page;
 import rx.subjects.PublishSubject;
 
 public class Download {
-    public Source source;
+    public OnlineSource source;
     public Manga manga;
     public Chapter chapter;
     public List<Page> pages;
@@ -29,7 +29,7 @@ public class Download {
     public static final int ERROR = 4;
 
 
-    public Download(Source source, Manga manga, Chapter chapter) {
+    public Download(OnlineSource source, Manga manga, Chapter chapter) {
         this.source = source;
         this.manga = manga;
         this.chapter = chapter;

app/src/main/java/eu/kanade/tachiyomi/data/glide/MangaModelLoader.kt

@@ -9,6 +9,7 @@ import eu.kanade.tachiyomi.App
 import eu.kanade.tachiyomi.data.cache.CoverCache
 import eu.kanade.tachiyomi.data.database.models.Manga
 import eu.kanade.tachiyomi.data.source.SourceManager
+import eu.kanade.tachiyomi.data.source.base.OnlineSource
 import java.io.File
 import java.io.InputStream
 import javax.inject.Inject
@@ -103,12 +104,11 @@ class MangaModelLoader(context: Context) : StreamModelLoader<Manga> {
      *
      * @param manga the model.
      */
-    fun getHeaders(manga: Manga): LazyHeaders {
+    fun getHeaders(manga: Manga): Headers {
+        val source = sourceManager.get(manga.source) as? OnlineSource ?: return LazyHeaders.DEFAULT
         return cachedHeaders.getOrPut(manga.source) {
-            val source = sourceManager.get(manga.source)!!
-
             LazyHeaders.Builder().apply {
-                for ((key, value) in source.requestHeaders.toMultimap()) {
+                for ((key, value) in source.headers.toMultimap()) {
                     addHeader(key, value[0])
                 }
             }.build()

app/src/main/java/eu/kanade/tachiyomi/data/library/LibraryUpdateService.kt

@@ -18,6 +18,7 @@ import eu.kanade.tachiyomi.data.database.models.Category
 import eu.kanade.tachiyomi.data.database.models.Manga
 import eu.kanade.tachiyomi.data.preference.PreferencesHelper
 import eu.kanade.tachiyomi.data.source.SourceManager
+import eu.kanade.tachiyomi.data.source.base.OnlineSource
 import eu.kanade.tachiyomi.ui.main.MainActivity
 import eu.kanade.tachiyomi.util.*
 import rx.Observable
@@ -288,9 +289,8 @@ class LibraryUpdateService : Service() {
      * @return a pair of the inserted and removed chapters.
      */
     fun updateManga(manga: Manga): Observable<Pair<Int, Int>> {
-        val source = sourceManager.get(manga.source)
-        return source!!
-                .pullChaptersFromNetwork(manga.url)
+        val source = sourceManager.get(manga.source) as? OnlineSource ?: return Observable.empty()
+        return source.fetchChapterList(manga)
                 .map { syncChaptersWithSource(db, it, manga, source) }
     }
 

app/src/main/java/eu/kanade/tachiyomi/data/network/CloudflareScraper.kt → app/src/main/java/eu/kanade/tachiyomi/data/network/CloudflareInterceptor.kt

@@ -6,7 +6,7 @@ import okhttp3.Interceptor
 import okhttp3.Request
 import okhttp3.Response
 
-object CloudflareScraper {
+class CloudflareInterceptor(private val cookies: PersistentCookieStore) : Interceptor {
 
     //language=RegExp
     private val operationPattern = Regex("""setTimeout\(function\(\)\{\s+(var t,r,a,f.+?\r?\n[\s\S]+?a\.value =.+?)\r?\n""")
@@ -17,7 +17,7 @@ object CloudflareScraper {
     //language=RegExp
     private val challengePattern = Regex("""name="jschl_vc" value="(\w+)"""")
 
-    fun request(chain: Interceptor.Chain, cookies: PersistentCookieStore): Response {
+    override fun intercept(chain: Interceptor.Chain): Response {
         val response = chain.proceed(chain.request())
 
         // Check if we already solved a challenge

app/src/main/java/eu/kanade/tachiyomi/data/network/NetworkHelper.kt

@@ -32,20 +32,18 @@ class NetworkHelper(context: Context) {
             .build()
 
     val cloudflareClient = defaultClient.newBuilder()
-            .addInterceptor { CloudflareScraper.request(it, cookies) }
+            .addInterceptor(CloudflareInterceptor(cookies))
             .build()
 
     val cookies: PersistentCookieStore
         get() = cookieManager.store
 
-    @JvmOverloads
     fun request(request: Request, client: OkHttpClient = defaultClient): Observable<Response> {
         return Observable.fromCallable {
-            client.newCall(request).execute().apply { body().close() }
+            client.newCall(request).execute()
         }
     }
 
-    @JvmOverloads
     fun requestBody(request: Request, client: OkHttpClient = defaultClient): Observable<String> {
         return Observable.fromCallable {
             client.newCall(request).execute().body().string()

app/src/main/java/eu/kanade/tachiyomi/data/network/Req.kt

@@ -1,9 +1,9 @@
 package eu.kanade.tachiyomi.data.network
 
 import okhttp3.*
-import java.util.concurrent.TimeUnit
+import java.util.concurrent.TimeUnit.MINUTES
 
-private val DEFAULT_CACHE_CONTROL = CacheControl.Builder().maxAge(10, TimeUnit.MINUTES).build()
+private val DEFAULT_CACHE_CONTROL = CacheControl.Builder().maxAge(10, MINUTES).build()
 private val DEFAULT_HEADERS = Headers.Builder().build()
 private val DEFAULT_BODY: RequestBody = FormBody.Builder().build()
 

app/src/main/java/eu/kanade/tachiyomi/data/source/SourceManager.kt

@@ -1,17 +1,19 @@
 package eu.kanade.tachiyomi.data.source
 
 import android.content.Context
+import android.os.Environment
+import eu.kanade.tachiyomi.R
+import eu.kanade.tachiyomi.data.source.base.OnlineSource
 import eu.kanade.tachiyomi.data.source.base.Source
-import eu.kanade.tachiyomi.data.source.online.english.*
-import eu.kanade.tachiyomi.data.source.online.russian.Mangachan
-import eu.kanade.tachiyomi.data.source.online.russian.Mintmanga
-import eu.kanade.tachiyomi.data.source.online.russian.Readmanga
-import java.util.*
+import eu.kanade.tachiyomi.data.source.base.YamlOnlineSource
+import eu.kanade.tachiyomi.data.source.online.english.Batoto
+import eu.kanade.tachiyomi.data.source.online.english.Kissmanga
+import org.yaml.snakeyaml.Yaml
+import timber.log.Timber
+import java.io.File
 
 open class SourceManager(private val context: Context) {
 
-    val sourcesMap: HashMap<Int, Source>
-
     val BATOTO = 1
     val MANGAHERE = 2
     val MANGAFOX = 3
@@ -23,38 +25,39 @@ open class SourceManager(private val context: Context) {
 
     val LAST_SOURCE = 8
 
-    init {
-        sourcesMap = createSourcesMap()
-    }
+    val sourcesMap = createSources()
 
     open fun get(sourceKey: Int): Source? {
         return sourcesMap[sourceKey]
     }
 
-    private fun createSource(sourceKey: Int): Source? = when (sourceKey) {
-        BATOTO -> Batoto(context)
-        MANGAHERE -> Mangahere(context)
-        MANGAFOX -> Mangafox(context)
-        KISSMANGA -> Kissmanga(context)
-        READMANGA -> Readmanga(context)
-        MINTMANGA -> Mintmanga(context)
-        MANGACHAN -> Mangachan(context)
-        READMANGATODAY -> ReadMangaToday(context)
+    fun getOnlineSources() = sourcesMap.values.filterIsInstance(OnlineSource::class.java)
+
+    private fun createSource(id: Int): Source? = when (id) {
+        BATOTO -> Batoto(context, id)
+        KISSMANGA -> Kissmanga(context, id)
         else -> null
     }
 
-    private fun createSourcesMap(): HashMap<Int, Source> {
-        val map = HashMap<Int, Source>()
+    private fun createSources(): Map<Int, Source> = hashMapOf<Int, Source>().apply {
         for (i in 1..LAST_SOURCE) {
-            val source = createSource(i)
-            if (source != null) {
-                source.id = i
-                map.put(i, source)
+            createSource(i)?.let { put(i, it) }
+        }
+
+        val parsersDir = File(Environment.getExternalStorageDirectory().absolutePath +
+                File.separator + context.getString(R.string.app_name), "parsers")
+
+        if (parsersDir.exists()) {
+            val yaml = Yaml()
+            for (file in parsersDir.listFiles().filter { it.extension == "yml" }) {
+                try {
+                    val map = file.inputStream().use { yaml.loadAs(it, Map::class.java) }
+                    YamlOnlineSource(context, map).let { put(it.id, it) }
+                } catch (e: Exception) {
+                    Timber.e("Error loading source from file. Bad format?")
+                }
             }
         }
-        return map
     }
 
-    fun getSources(): List<Source> = ArrayList(sourcesMap.values)
-
 }

app/src/main/java/eu/kanade/tachiyomi/data/source/base/BaseSource.java

@@ -1,99 +0,0 @@
-package eu.kanade.tachiyomi.data.source.base;
-
-import org.jsoup.nodes.Document;
-
-import java.util.List;
-
-import eu.kanade.tachiyomi.data.database.models.Chapter;
-import eu.kanade.tachiyomi.data.database.models.Manga;
-import eu.kanade.tachiyomi.data.source.Language;
-import eu.kanade.tachiyomi.data.source.model.MangasPage;
-import okhttp3.Headers;
-import okhttp3.Response;
-import rx.Observable;
-
-public abstract class BaseSource {
-
-    private int id;
-
-    // Id of the source
-    public int getId() {
-        return id;
-    }
-
-    public void setId(int id) {
-        this.id = id;
-    }
-
-    public abstract Language getLang();
-
-    // Name of the source to display
-    public abstract String getName();
-
-    // Name of the source to display with the language
-    public String getVisibleName() {
-        return getName() + " (" + getLang().getCode() + ")";
-    }
-
-    // Base url of the source, like: http://example.com
-    public abstract String getBaseUrl();
-
-    // True if the source requires a login
-    public abstract boolean isLoginRequired();
-
-    // Return the initial popular mangas URL
-    protected abstract String getInitialPopularMangasUrl();
-
-    // Return the initial search url given a query
-    protected abstract String getInitialSearchUrl(String query);
-
-    // Get the popular list of mangas from the source's parsed document
-    protected abstract List<Manga> parsePopularMangasFromHtml(Document parsedHtml);
-
-    // Get the next popular page URL or null if it's the last
-    protected abstract String parseNextPopularMangasUrl(Document parsedHtml, MangasPage page);
-
-    // Get the searched list of mangas from the source's parsed document
-    protected abstract List<Manga> parseSearchFromHtml(Document parsedHtml);
-
-    // Get the next search page URL or null if it's the last
-    protected abstract String parseNextSearchUrl(Document parsedHtml, MangasPage page, String query);
-
-    // Given the URL of a manga and the result of the request, return the details of the manga
-    protected abstract Manga parseHtmlToManga(String mangaUrl, String unparsedHtml);
-
-    // Given the result of the request to mangas' chapters, return a list of chapters
-    protected abstract List<Chapter> parseHtmlToChapters(String unparsedHtml);
-
-    // Given the result of the request to a chapter, return the list of URLs of the chapter
-    protected abstract List<String> parseHtmlToPageUrls(String unparsedHtml);
-
-    // Given the result of the request to a chapter's page, return the URL of the image of the page
-    protected abstract String parseHtmlToImageUrl(String unparsedHtml);
-
-
-    // Login related methods, shouldn't be overriden if the source doesn't require it
-    public Observable<Boolean> login(String username, String password) {
-        throw new UnsupportedOperationException("Not implemented");
-    }
-
-    public boolean isLogged() {
-        throw new UnsupportedOperationException("Not implemented");
-    }
-
-    protected boolean isAuthenticationSuccessful(Response response) {
-        throw new UnsupportedOperationException("Not implemented");
-    }
-
-    // Default headers, it can be overriden by children or just add new keys
-    protected Headers.Builder headersBuilder() {
-        Headers.Builder builder = new Headers.Builder();
-        builder.add("User-Agent", "Mozilla/5.0 (Windows NT 6.3; WOW64)");
-        return builder;
-    }
-
-    @Override
-    public String toString() {
-        return getVisibleName();
-    }
-}

app/src/main/java/eu/kanade/tachiyomi/data/source/base/LoginSource.java

@@ -1,15 +0,0 @@
-package eu.kanade.tachiyomi.data.source.base;
-
-import android.content.Context;
-
-public abstract class LoginSource extends Source {
-
-    public LoginSource(Context context) {
-        super(context);
-    }
-
-    @Override
-    public boolean isLoginRequired() {
-        return true;
-    }
-}

app/src/main/java/eu/kanade/tachiyomi/data/source/base/OnlineSource.kt

@@ -0,0 +1,448 @@
+package eu.kanade.tachiyomi.data.source.base
+
+import android.content.Context
+import eu.kanade.tachiyomi.App
+import eu.kanade.tachiyomi.data.cache.ChapterCache
+import eu.kanade.tachiyomi.data.database.models.Chapter
+import eu.kanade.tachiyomi.data.database.models.Manga
+import eu.kanade.tachiyomi.data.network.NetworkHelper
+import eu.kanade.tachiyomi.data.network.get
+import eu.kanade.tachiyomi.data.preference.PreferencesHelper
+import eu.kanade.tachiyomi.data.source.Language
+import eu.kanade.tachiyomi.data.source.model.MangasPage
+import eu.kanade.tachiyomi.data.source.model.Page
+import okhttp3.*
+import rx.Observable
+import javax.inject.Inject
+
+/**
+ * A simple implementation for sources from a website.
+ *
+ * @param context the application context.
+ */
+abstract class OnlineSource(context: Context) : Source {
+
+    /**
+     * Network service.
+     */
+    @Inject lateinit var network: NetworkHelper
+
+    /**
+     * Chapter cache.
+     */
+    @Inject lateinit var chapterCache: ChapterCache
+
+    /**
+     * Preferences helper.
+     */
+    @Inject lateinit var preferences: PreferencesHelper
+
+    /**
+     * Base url of the website without the trailing slash, like: http://mysite.com
+     */
+    abstract val baseUrl: String
+
+    /**
+     * Language of the source.
+     */
+    abstract val lang: Language
+
+    /**
+     * Headers used for requests.
+     */
+    val headers by lazy { headersBuilder().build() }
+
+    /**
+     * Default network client for doing requests.
+     */
+    open val client: OkHttpClient
+        get() = network.defaultClient
+
+    init {
+        // Inject dependencies.
+        App.get(context).component.inject(this)
+    }
+
+    /**
+     * Headers builder for requests. Implementations can override this method for custom headers.
+     */
+    open protected fun headersBuilder() = Headers.Builder().apply {
+        add("User-Agent", "Mozilla/5.0 (Windows NT 6.3; WOW64)")
+    }
+
+    /**
+     * Visible name of the source.
+     */
+    override fun toString() = "$name (${lang.code})"
+
+    // Login source
+
+    open fun isLoginRequired() = false
+
+    open fun isLogged(): Boolean = throw Exception("Not implemented")
+
+    open fun login(username: String, password: String): Observable<Boolean>
+            = throw Exception("Not implemented")
+
+    open fun isAuthenticationSuccessful(response: Response): Boolean
+            = throw Exception("Not implemented")
+
+    /**
+     * Returns an observable containing a page with a list of manga. Normally it's not needed to
+     * override this method.
+     *
+     * @param page the page object where the information will be saved, like the list of manga,
+     *             the current page and the next page url.
+     */
+    open fun fetchPopularManga(page: MangasPage): Observable<MangasPage> = network
+            .request(popularMangaRequest(page), client)
+            .map { response ->
+                page.apply {
+                    mangas = mutableListOf<Manga>()
+                    popularMangaParse(response, this)
+                }
+            }
+
+    /**
+     * Returns the request for the popular manga given the page. Override only if it's needed to
+     * send different headers or request method like POST.
+     *
+     * @param page the page object.
+     */
+    open protected fun popularMangaRequest(page: MangasPage): Request {
+        if (page.page == 1) {
+            page.url = popularMangaInitialUrl()
+        }
+        return get(page.url, headers)
+    }
+
+    /**
+     * Returns the absolute url of the first page to popular manga.
+     */
+    abstract protected fun popularMangaInitialUrl(): String
+
+    /**
+     * Parse the response from the site. It should add a list of manga and the absolute url to the
+     * next page (if it has a next one) to [page].
+     *
+     * @param response the response from the site.
+     * @param page the page object to be filled.
+     */
+    abstract protected fun popularMangaParse(response: Response, page: MangasPage)
+
+    /**
+     * Returns an observable containing a page with a list of manga. Normally it's not needed to
+     * override this method.
+     *
+     * @param page the page object where the information will be saved, like the list of manga,
+     *             the current page and the next page url.
+     * @param query the search query.
+     */
+    open fun fetchSearchManga(page: MangasPage, query: String): Observable<MangasPage> = network
+            .request(searchMangaRequest(page, query), client)
+            .map { response ->
+                page.apply {
+                    mangas = mutableListOf<Manga>()
+                    searchMangaParse(response, this, query)
+                }
+            }
+
+    /**
+     * Returns the request for the search manga given the page. Override only if it's needed to
+     * send different headers or request method like POST.
+     *
+     * @param page the page object.
+     * @param query the search query.
+     */
+    open protected fun searchMangaRequest(page: MangasPage, query: String): Request {
+        if (page.page == 1) {
+            page.url = searchMangaInitialUrl(query)
+        }
+        return get(page.url, headers)
+    }
+
+    /**
+     * Returns the absolute url of the first page to popular manga.
+     *
+     * @param query the search query.
+     */
+    abstract protected fun searchMangaInitialUrl(query: String): String
+
+    /**
+     * Parse the response from the site. It should add a list of manga and the absolute url to the
+     * next page (if it has a next one) to [page].
+     *
+     * @param response the response from the site.
+     * @param page the page object to be filled.
+     * @param query the search query.
+     */
+    abstract protected fun searchMangaParse(response: Response, page: MangasPage, query: String)
+
+    /**
+     * Returns an observable with the updated details for a manga. Normally it's not needed to
+     * override this method.
+     *
+     * @param manga the manga to be updated.
+     */
+    override fun fetchMangaDetails(manga: Manga): Observable<Manga> = network
+            .request(mangaDetailsRequest(manga), client)
+            .map { response ->
+                Manga.create(manga.url, id).apply {
+                    mangaDetailsParse(response, this)
+                    initialized = true
+                }
+            }
+
+    /**
+     * Returns the request for updating a manga. Override only if it's needed to override the url,
+     * send different headers or request method like POST.
+     *
+     * @param manga the manga to be updated.
+     */
+    open protected fun mangaDetailsRequest(manga: Manga): Request {
+        return get(baseUrl + manga.url, headers)
+    }
+
+    /**
+     * Parse the response from the site. It should fill [manga].
+     *
+     * @param response the response from the site.
+     * @param manga the manga whose fields have to be filled.
+     */
+    abstract protected fun mangaDetailsParse(response: Response, manga: Manga)
+
+    /**
+     * Returns an observable with the updated chapter list for a manga. Normally it's not needed to
+     * override this method.
+     *
+     * @param manga the manga to look for chapters.
+     */
+    override fun fetchChapterList(manga: Manga): Observable<List<Chapter>> = network
+            .request(chapterListRequest(manga), client)
+            .map { response ->
+                mutableListOf<Chapter>().apply {
+                    chapterListParse(response, this)
+                    if (isEmpty()) {
+                        throw Exception("No chapters found")
+                    }
+                }
+            }
+
+    /**
+     * Returns the request for updating the chapter list. Override only if it's needed to override
+     * the url, send different headers or request method like POST.
+     *
+     * @param manga the manga to look for chapters.
+     */
+    open protected fun chapterListRequest(manga: Manga): Request {
+        return get(baseUrl + manga.url, headers)
+    }
+
+    /**
+     * Parse the response from the site. It should fill [chapters].
+     *
+     * @param response the response from the site.
+     * @param chapters the chapter list to be filled.
+     */
+    abstract protected fun chapterListParse(response: Response, chapters: MutableList<Chapter>)
+
+    /**
+     * Returns an observable with the page list for a chapter. It tries to return the page list from
+     * the local cache, otherwise fallbacks to network calling [fetchPageListFromNetwork].
+     *
+     * @param chapter the chapter whose page list has to be fetched.
+     */
+    final override fun fetchPageList(chapter: Chapter): Observable<List<Page>> = chapterCache
+            .getPageListFromCache(getChapterCacheKey(chapter))
+            .onErrorResumeNext { fetchPageListFromNetwork(chapter) }
+
+    /**
+     * Returns an observable with the page list for a chapter. Normally it's not needed to override
+     * this method.
+     *
+     * @param chapter the chapter whose page list has to be fetched.
+     */
+    open fun fetchPageListFromNetwork(chapter: Chapter): Observable<List<Page>> = network
+            .request(pageListRequest(chapter), client)
+            .map { response ->
+                mutableListOf<Page>().apply {
+                    pageListParse(response, this)
+                    if (isEmpty()) {
+                        throw Exception("Page list is empty")
+                    }
+                }
+            }
+
+    /**
+     * Returns the request for getting the page list. Override only if it's needed to override the
+     * url, send different headers or request method like POST.
+     *
+     * @param chapter the chapter whose page list has to be fetched
+     */
+    open protected fun pageListRequest(chapter: Chapter): Request {
+        return get(baseUrl + chapter.url, headers)
+    }
+
+    /**
+     * Parse the response from the site. It should fill [pages].
+     *
+     * @param response the response from the site.
+     * @param pages the page list to be filled.
+     */
+    abstract protected fun pageListParse(response: Response, pages: MutableList<Page>)
+
+    /**
+     * Returns the key for the page list to be stored in [ChapterCache].
+     */
+    private fun getChapterCacheKey(chapter: Chapter) = "$id${chapter.url}"
+
+    /**
+     * Returns an observable with the page containing the source url of the image. If there's any
+     * error, it will return null instead of throwing an exception.
+     *
+     * @param page the page whose source image has to be fetched.
+     */
+    open protected fun fetchImageUrl(page: Page): Observable<Page> {
+        page.status = Page.LOAD_PAGE
+        return network
+                .request(imageUrlRequest(page), client)
+                .map { imageUrlParse(it) }
+                .doOnError { page.status = Page.ERROR }
+                .onErrorReturn { null }
+                .doOnNext { page.imageUrl = it }
+                .map { page }
+    }
+
+    /**
+     * Returns the request for getting the url to the source image. Override only if it's needed to
+     * override the url, send different headers or request method like POST.
+     *
+     * @param page the chapter whose page list has to be fetched
+     */
+    open protected fun imageUrlRequest(page: Page): Request {
+        return get(page.url, headers)
+    }
+
+    /**
+     * Parse the response from the site. It should return the absolute url to the source image.
+     *
+     * @param response the response from the site.
+     */
+    abstract protected fun imageUrlParse(response: Response): String
+
+    /**
+     * Returns an observable of the page with the downloaded image.
+     *
+     * @param page the page whose source image has to be downloaded.
+     */
+    final override fun fetchImage(page: Page): Observable<Page> =
+        if (page.imageUrl.isNullOrEmpty())
+            fetchImageUrl(page).flatMap { getCachedImage(it) }
+        else
+            getCachedImage(page)
+
+    /**
+     * Returns an observable with the response of the source image.
+     *
+     * @param page the page whose source image has to be downloaded.
+     */
+    fun imageResponse(page: Page): Observable<Response> = network
+            .requestBodyProgress(imageRequest(page), page)
+            .doOnNext {
+                if (!it.isSuccessful) {
+                    it.body().close()
+                    throw RuntimeException("Not a valid response")
+                }
+            }
+
+    /**
+     * Returns the request for getting the source image. Override only if it's needed to override
+     * the url, send different headers or request method like POST.
+     *
+     * @param page the chapter whose page list has to be fetched
+     */
+    open protected fun imageRequest(page: Page): Request {
+        return get(page.imageUrl, headers)
+    }
+
+    /**
+     * Returns an observable of the page that gets the image from the chapter or fallbacks to
+     * network and copies it to the cache calling [cacheImage].
+     *
+     * @param page the page.
+     */
+    fun getCachedImage(page: Page): Observable<Page> {
+        val pageObservable = Observable.just(page)
+        if (page.imageUrl.isNullOrEmpty())
+            return pageObservable
+
+        return pageObservable
+                .flatMap {
+                    if (!chapterCache.isImageInCache(page.imageUrl)) {
+                        cacheImage(page)
+                    } else {
+                        Observable.just(page)
+                    }
+                }
+                .doOnNext {
+                    page.imagePath = chapterCache.getImagePath(page.imageUrl)
+                    page.status = Page.READY
+                }
+                .doOnError { page.status = Page.ERROR }
+                .onErrorReturn { page }
+    }
+
+    /**
+     * Returns an observable of the page that downloads the image to [ChapterCache].
+     *
+     * @param page the page.
+     */
+    private fun cacheImage(page: Page): Observable<Page> {
+        page.status = Page.DOWNLOAD_IMAGE
+        return imageResponse(page)
+                .doOnNext { chapterCache.putImageToCache(page.imageUrl, it, preferences.reencodeImage()) }
+                .map { page }
+    }
+
+
+    // Utility methods
+
+    /**
+     * Returns an absolute url from a href.
+     *
+     * Ex:
+     * href="http://example.com/foo" url="http://example.com" -> http://example.com/foo
+     * href="/mypath" url="http://example.com/foo" -> http://example.com/mypath
+     * href="bar" url="http://example.com/foo" -> http://example.com/bar
+     * href="bar" url="http://example.com/foo/" -> http://example.com/foo/bar
+     *
+     * @param href the href attribute from the html.
+     * @param url the requested url.
+     */
+    fun getAbsoluteUrl(href: String, url: HttpUrl) = when {
+        href.startsWith("http://") || href.startsWith("https://") -> href
+        href.startsWith("/") -> url.newBuilder().encodedPath("/").fragment(null).query(null)
+                .toString() + href.substring(1)
+        else -> url.toString().substringBeforeLast('/') + "/$href"
+    }
+
+    fun fetchAllImageUrlsFromPageList(pages: List<Page>) = Observable.from(pages)
+            .filter { !it.imageUrl.isNullOrEmpty() }
+            .mergeWith(fetchRemainingImageUrlsFromPageList(pages))
+
+    fun fetchRemainingImageUrlsFromPageList(pages: List<Page>) = Observable.from(pages)
+            .filter { it.imageUrl.isNullOrEmpty() }
+            .concatMap { fetchImageUrl(it) }
+
+    fun savePageList(chapter: Chapter, pages: List<Page>?) {
+        if (pages != null) {
+            chapterCache.putPageListToCache(getChapterCacheKey(chapter), pages)
+        }
+    }
+
+    // Overridable method to allow custom parsing.
+    open fun parseChapterNumber(chapter: Chapter) {
+
+    }
+
+}

app/src/main/java/eu/kanade/tachiyomi/data/source/base/ParsedOnlineSource.kt

@@ -0,0 +1,189 @@
+package eu.kanade.tachiyomi.data.source.base
+
+import android.content.Context
+import eu.kanade.tachiyomi.data.database.models.Chapter
+import eu.kanade.tachiyomi.data.database.models.Manga
+import eu.kanade.tachiyomi.data.source.model.MangasPage
+import eu.kanade.tachiyomi.data.source.model.Page
+import okhttp3.Response
+import org.jsoup.Jsoup
+import org.jsoup.nodes.Document
+import org.jsoup.nodes.Element
+
+/**
+ * A simple implementation for sources from a website using Jsoup, an HTML parser.
+ *
+ * @param context the application context.
+ */
+abstract class ParsedOnlineSource(context: Context) : OnlineSource(context) {
+
+    /**
+     * Parse the response from the site and fills [page].
+     *
+     * @param response the response from the site.
+     * @param page the page object to be filled.
+     */
+    override fun popularMangaParse(response: Response, page: MangasPage) {
+        val document = Jsoup.parse(response.body().string())
+        for (element in document.select(popularMangaSelector())) {
+            Manga().apply {
+                source = [email protected]
+                popularMangaFromElement(element, this)
+                page.mangas.add(this)
+            }
+        }
+
+        popularMangaNextPageSelector()?.let { selector ->
+            page.nextPageUrl = document.select(selector).first()?.attr("href")?.let {
+                getAbsoluteUrl(it, response.request().url())
+            }
+        }
+    }
+
+    /**
+     * Returns the Jsoup selector that returns a list of [Element] corresponding to each manga.
+     */
+    abstract protected fun popularMangaSelector(): String
+
+    /**
+     * Fills [manga] with the given [element]. Most sites only show the title and the url, it's
+     * totally safe to fill only those two values.
+     *
+     * @param element an element obtained from [popularMangaSelector].
+     * @param manga the manga to fill.
+     */
+    abstract protected fun popularMangaFromElement(element: Element, manga: Manga)
+
+    /**
+     * Returns the Jsoup selector that returns the <a> tag linking to the next page, or null if
+     * there's no next page.
+     */
+    abstract protected fun popularMangaNextPageSelector(): String?
+
+    /**
+     * Parse the response from the site and fills [page].
+     *
+     * @param response the response from the site.
+     * @param page the page object to be filled.
+     * @param query the search query.
+     */
+    override fun searchMangaParse(response: Response, page: MangasPage, query: String) {
+        val document = Jsoup.parse(response.body().string())
+        for (element in document.select(searchMangaSelector())) {
+            Manga().apply {
+                source = [email protected]
+                searchMangaFromElement(element, this)
+                page.mangas.add(this)
+            }
+        }
+
+        searchMangaNextPageSelector()?.let { selector ->
+            page.nextPageUrl = document.select(selector).first()?.attr("href")?.let {
+                getAbsoluteUrl(it, response.request().url())
+            }
+        }
+    }
+
+    /**
+     * Returns the Jsoup selector that returns a list of [Element] corresponding to each manga.
+     */
+    abstract protected fun searchMangaSelector(): String
+
+    /**
+     * Fills [manga] with the given [element]. Most sites only show the title and the url, it's
+     * totally safe to fill only those two values.
+     *
+     * @param element an element obtained from [searchMangaSelector].
+     * @param manga the manga to fill.
+     */
+    abstract protected fun searchMangaFromElement(element: Element, manga: Manga)
+
+    /**
+     * Returns the Jsoup selector that returns the <a> tag linking to the next page, or null if
+     * there's no next page.
+     */
+    abstract protected fun searchMangaNextPageSelector(): String?
+
+    /**
+     * Parse the response from the site and fills the details of [manga].
+     *
+     * @param response the response from the site.
+     * @param manga the manga to fill.
+     */
+    override fun mangaDetailsParse(response: Response, manga: Manga) {
+        mangaDetailsParse(Jsoup.parse(response.body().string()), manga)
+    }
+
+    /**
+     * Fills the details of [manga] from the given [document].
+     *
+     * @param document the parsed document.
+     * @param manga the manga to fill.
+     */
+    abstract protected fun mangaDetailsParse(document: Document, manga: Manga)
+
+    /**
+     * Parse the response from the site and fills the chapter list.
+     *
+     * @param response the response from the site.
+     * @param chapters the list of chapters to fill.
+     */
+    override fun chapterListParse(response: Response, chapters: MutableList<Chapter>) {
+        val document = Jsoup.parse(response.body().string())
+
+        for (element in document.select(chapterListSelector())) {
+            Chapter.create().apply {
+                chapterFromElement(element, this)
+                chapters.add(this)
+            }
+        }
+    }
+
+    /**
+     * Returns the Jsoup selector that returns a list of [Element] corresponding to each chapter.
+     */
+    abstract protected fun chapterListSelector(): String
+
+    /**
+     * Fills [chapter] with the given [element].
+     *
+     * @param element an element obtained from [chapterListSelector].
+     * @param chapter the chapter to fill.
+     */
+    abstract protected fun chapterFromElement(element: Element, chapter: Chapter)
+
+    /**
+     * Parse the response from the site and fills the page list.
+     *
+     * @param response the response from the site.
+     * @param pages the list of pages to fill.
+     */
+    override fun pageListParse(response: Response, pages: MutableList<Page>) {
+        pageListParse(Jsoup.parse(response.body().string()), pages)
+    }
+
+    /**
+     * Fills [pages] from the given [document].
+     *
+     * @param document the parsed document.
+     * @param pages the list of pages to fill.
+     */
+    abstract protected fun pageListParse(document: Document, pages: MutableList<Page>)
+
+    /**
+     * Parse the response from the site and returns the absolute url to the source image.
+     *
+     * @param response the response from the site.
+     */
+    override fun imageUrlParse(response: Response): String {
+        return imageUrlParse(Jsoup.parse(response.body().string()))
+    }
+
+    /**
+     * Returns the absolute url to the source image from the document.
+     *
+     * @param document the parsed document.
+     */
+    abstract protected fun imageUrlParse(document: Document): String
+
+}

app/src/main/java/eu/kanade/tachiyomi/data/source/base/Source.kt

@@ -1,228 +1,51 @@
 package eu.kanade.tachiyomi.data.source.base
 
-import android.content.Context
-import eu.kanade.tachiyomi.App
-import eu.kanade.tachiyomi.data.cache.ChapterCache
 import eu.kanade.tachiyomi.data.database.models.Chapter
 import eu.kanade.tachiyomi.data.database.models.Manga
-import eu.kanade.tachiyomi.data.network.NetworkHelper
-import eu.kanade.tachiyomi.data.network.get
-import eu.kanade.tachiyomi.data.preference.PreferencesHelper
-import eu.kanade.tachiyomi.data.source.model.MangasPage
 import eu.kanade.tachiyomi.data.source.model.Page
-import okhttp3.OkHttpClient
-import okhttp3.Request
-import okhttp3.Response
-import org.jsoup.Jsoup
 import rx.Observable
-import rx.schedulers.Schedulers
-import java.util.*
-import javax.inject.Inject
 
-abstract class Source(context: Context) : BaseSource() {
-
-    @Inject protected lateinit var networkService: NetworkHelper
-    @Inject protected lateinit var chapterCache: ChapterCache
-    @Inject protected lateinit var prefs: PreferencesHelper
-
-    val requestHeaders by lazy { headersBuilder().build() }
-
-    init {
-        App.get(context).component.inject(this)
-    }
-
-    open val networkClient: OkHttpClient
-        get() = networkService.defaultClient
-
-    override fun isLoginRequired(): Boolean {
-        return false
-    }
-
-    protected fun popularMangaRequest(page: MangasPage): Request {
-        if (page.page == 1) {
-            page.url = initialPopularMangasUrl
-        }
-
-        return get(page.url, requestHeaders)
-    }
-
-    protected open fun searchMangaRequest(page: MangasPage, query: String): Request {
-        if (page.page == 1) {
-            page.url = getInitialSearchUrl(query)
-        }
-
-        return get(page.url, requestHeaders)
-    }
-
-    protected open fun mangaDetailsRequest(mangaUrl: String): Request {
-        return get(baseUrl + mangaUrl, requestHeaders)
-    }
-
-    protected fun chapterListRequest(mangaUrl: String): Request {
-        return get(baseUrl + mangaUrl, requestHeaders)
-    }
-
-    protected open fun pageListRequest(chapterUrl: String): Request {
-        return get(baseUrl + chapterUrl, requestHeaders)
-    }
-
-    protected open fun imageUrlRequest(page: Page): Request {
-        return get(page.url, requestHeaders)
-    }
-
-    protected open fun imageRequest(page: Page): Request {
-        return get(page.imageUrl, requestHeaders)
-    }
-
-    // Get the most popular mangas from the source
-    open fun pullPopularMangasFromNetwork(page: MangasPage): Observable<MangasPage> {
-        return networkService.requestBody(popularMangaRequest(page), networkClient)
-                .map { Jsoup.parse(it) }
-                .doOnNext { doc -> page.mangas = parsePopularMangasFromHtml(doc) }
-                .doOnNext { doc -> page.nextPageUrl = parseNextPopularMangasUrl(doc, page) }
-                .map { response -> page }
-    }
-
-    // Get mangas from the source with a query
-    open fun searchMangasFromNetwork(page: MangasPage, query: String): Observable<MangasPage> {
-        return networkService.requestBody(searchMangaRequest(page, query), networkClient)
-                .map { Jsoup.parse(it) }
-                .doOnNext { doc -> page.mangas = parseSearchFromHtml(doc) }
-                .doOnNext { doc -> page.nextPageUrl = parseNextSearchUrl(doc, page, query) }
-                .map { response -> page }
-    }
-
-    // Get manga details from the source
-    open fun pullMangaFromNetwork(mangaUrl: String): Observable<Manga> {
-        return networkService.requestBody(mangaDetailsRequest(mangaUrl), networkClient)
-                .flatMap { Observable.just(parseHtmlToManga(mangaUrl, it)) }
-    }
-
-    // Get chapter list of a manga from the source
-    open fun pullChaptersFromNetwork(mangaUrl: String): Observable<List<Chapter>> {
-        return networkService.requestBody(chapterListRequest(mangaUrl), networkClient)
-                .flatMap { unparsedHtml ->
-                    val chapters = parseHtmlToChapters(unparsedHtml)
-                    if (!chapters.isEmpty())
-                        Observable.just(chapters)
-                    else
-                        Observable.error(Exception("No chapters found"))
-                }
-    }
-
-    open fun getCachedPageListOrPullFromNetwork(chapterUrl: String): Observable<List<Page>> {
-        return chapterCache.getPageListFromCache(getChapterCacheKey(chapterUrl))
-                .onErrorResumeNext { pullPageListFromNetwork(chapterUrl) }
-                .onBackpressureBuffer()
-    }
-
-    open fun pullPageListFromNetwork(chapterUrl: String): Observable<List<Page>> {
-        return networkService.requestBody(pageListRequest(chapterUrl), networkClient)
-                .flatMap { unparsedHtml ->
-                    val pages = convertToPages(parseHtmlToPageUrls(unparsedHtml))
-                    if (!pages.isEmpty())
-                        Observable.just(parseFirstPage(pages, unparsedHtml))
-                    else
-                        Observable.error(Exception("Page list is empty"))
-                }
-    }
-
-    open fun getAllImageUrlsFromPageList(pages: List<Page>): Observable<Page> {
-        return Observable.from(pages)
-                .filter { page -> page.imageUrl != null }
-                .mergeWith(getRemainingImageUrlsFromPageList(pages))
-    }
-
-    // Get the URLs of the images of a chapter
-    open fun getRemainingImageUrlsFromPageList(pages: List<Page>): Observable<Page> {
-        return Observable.from(pages)
-                .filter { page -> page.imageUrl == null }
-                .concatMap { getImageUrlFromPage(it) }
-    }
-
-    open fun getImageUrlFromPage(page: Page): Observable<Page> {
-        page.status = Page.LOAD_PAGE
-        return networkService.requestBody(imageUrlRequest(page), networkClient)
-                .flatMap { unparsedHtml -> Observable.just(parseHtmlToImageUrl(unparsedHtml)) }
-                .onErrorResumeNext { e ->
-                    page.status = Page.ERROR
-                    Observable.just<String>(null)
-                }
-                .flatMap { imageUrl ->
-                    page.imageUrl = imageUrl
-                    Observable.just(page)
-                }
-                .subscribeOn(Schedulers.io())
-    }
-
-    open fun getCachedImage(page: Page): Observable<Page> {
-        val pageObservable = Observable.just(page)
-        if (page.imageUrl == null)
-            return pageObservable
-
-        return pageObservable
-                .flatMap { p ->
-                    if (!chapterCache.isImageInCache(page.imageUrl)) {
-                        return@flatMap cacheImage(page)
-                    }
-                    Observable.just(page)
-                }
-                .flatMap { p ->
-                    page.imagePath = chapterCache.getImagePath(page.imageUrl)
-                    page.status = Page.READY
-                    Observable.just(page)
-                }
-                .onErrorResumeNext { e ->
-                    page.status = Page.ERROR
-                    Observable.just(page)
-                }
-    }
-
-    private fun cacheImage(page: Page): Observable<Page> {
-        page.status = Page.DOWNLOAD_IMAGE
-        return getImageProgressResponse(page)
-                .flatMap { resp ->
-                    chapterCache.putImageToCache(page.imageUrl, resp, prefs.reencodeImage())
-                    Observable.just(page)
-                }
-    }
-
-    open fun getImageProgressResponse(page: Page): Observable<Response> {
-        return networkService.requestBodyProgress(imageRequest(page), page)
-                .doOnNext {
-                    if (!it.isSuccessful) {
-                        it.body().close()
-                        throw RuntimeException("Not a valid response")
-                    }
-                }
-    }
-
-    fun savePageList(chapterUrl: String, pages: List<Page>?) {
-        if (pages != null)
-            chapterCache.putPageListToCache(getChapterCacheKey(chapterUrl), pages)
-    }
-
-    protected open fun convertToPages(pageUrls: List<String>): List<Page> {
-        val pages = ArrayList<Page>()
-        for (i in pageUrls.indices) {
-            pages.add(Page(i, pageUrls[i]))
-        }
-        return pages
-    }
-
-    protected open fun parseFirstPage(pages: List<Page>, unparsedHtml: String): List<Page> {
-        val firstImage = parseHtmlToImageUrl(unparsedHtml)
-        pages[0].imageUrl = firstImage
-        return pages
-    }
-
-    protected fun getChapterCacheKey(chapterUrl: String): String {
-        return "$id$chapterUrl"
-    }
-
-    // Overridable method to allow custom parsing.
-    open fun parseChapterNumber(chapter: Chapter) {
-
-    }
-
-}
+/**
+ * A basic interface for creating a source. It could be an online source, a local source, etc...
+ */
+interface Source {
+
+    /**
+     * Id for the source. Must be unique.
+     */
+    val id: Int
+
+    /**
+     * Name of the source.
+     */
+    val name: String
+
+    /**
+     * Returns an observable with the updated details for a manga.
+     *
+     * @param manga the manga to update.
+     */
+    fun fetchMangaDetails(manga: Manga): Observable<Manga>
+
+    /**
+     * Returns an observable with all the available chapters for a manga.
+     *
+     * @param manga the manga to update.
+     */
+    fun fetchChapterList(manga: Manga): Observable<List<Chapter>>
+
+    /**
+     * Returns an observable with the list of pages a chapter has.
+     *
+     * @param chapter the chapter.
+     */
+    fun fetchPageList(chapter: Chapter): Observable<List<Page>>
+
+    /**
+     * Returns an observable with the path of the image.
+     *
+     * @param page the page.
+     */
+    fun fetchImage(page: Page): Observable<Page>
+
+}

app/src/main/java/eu/kanade/tachiyomi/data/source/base/YamlOnlineSource.kt

@@ -0,0 +1,166 @@
+package eu.kanade.tachiyomi.data.source.base
+
+import android.content.Context
+import eu.kanade.tachiyomi.data.database.models.Chapter
+import eu.kanade.tachiyomi.data.database.models.Manga
+import eu.kanade.tachiyomi.data.network.get
+import eu.kanade.tachiyomi.data.network.post
+import eu.kanade.tachiyomi.data.source.getLanguages
+import eu.kanade.tachiyomi.data.source.model.MangasPage
+import eu.kanade.tachiyomi.data.source.model.Page
+import okhttp3.Request
+import okhttp3.Response
+import org.jsoup.Jsoup
+import org.jsoup.nodes.Element
+import java.text.SimpleDateFormat
+import java.util.*
+
+class YamlOnlineSource(context: Context, mappings: Map<*, *>) : OnlineSource(context) {
+
+    val map = YamlSourceNode(mappings)
+
+    override val name: String
+        get() = map.name
+
+    override val baseUrl = map.host.let {
+        if (it.endsWith("/")) it.dropLast(1) else it
+    }
+
+    override val lang = map.lang.toUpperCase().let { code ->
+        getLanguages().find { code == it.code }!!
+    }
+
+    override val client = when(map.client) {
+        "cloudflare" -> network.cloudflareClient
+        else -> network.defaultClient
+    }
+
+    override val id = map.id.let {
+        if (it is Int) it else (lang.code.hashCode() + 31 * it.hashCode()) and 0x7fffffff
+    }
+
+    override fun popularMangaRequest(page: MangasPage): Request {
+        if (page.page == 1) {
+            page.url = popularMangaInitialUrl()
+        }
+        return when (map.popular.method?.toLowerCase()) {
+            "post" -> post(page.url, headers, map.popular.createForm())
+            else -> get(page.url, headers)
+        }
+    }
+
+    override fun popularMangaInitialUrl() = map.popular.url
+
+    override fun popularMangaParse(response: Response, page: MangasPage) {
+        val document = Jsoup.parse(response.body().string())
+        for (element in document.select(map.popular.manga_css)) {
+            Manga().apply {
+                source = [email protected]
+                title = element.text()
+                setUrl(element.attr("href"))
+                page.mangas.add(this)
+            }
+        }
+
+        map.popular.next_url_css?.let { selector ->
+            page.nextPageUrl = document.select(selector).first()?.attr("href")?.let {
+                getAbsoluteUrl(it, response.request().url())
+            }
+        }
+    }
+
+    override fun searchMangaRequest(page: MangasPage, query: String): Request {
+        if (page.page == 1) {
+            page.url = searchMangaInitialUrl(query)
+        }
+        return when (map.search.method?.toLowerCase()) {
+            "post" -> post(page.url, headers, map.search.createForm())
+            else -> get(page.url, headers)
+        }
+    }
+
+    override fun searchMangaInitialUrl(query: String) = map.search.url.replace("\$query", query)
+
+    override fun searchMangaParse(response: Response, page: MangasPage, query: String) {
+        val document = Jsoup.parse(response.body().string())
+        for (element in document.select(map.search.manga_css)) {
+            Manga().apply {
+                source = [email protected]
+                title = element.text()
+                setUrl(element.attr("href"))
+                page.mangas.add(this)
+            }
+        }
+
+        map.search.next_url_css?.let { selector ->
+            page.nextPageUrl = document.select(selector).first()?.attr("href")?.let {
+                getAbsoluteUrl(it, response.request().url())
+            }
+        }
+    }
+
+    override fun mangaDetailsParse(response: Response, manga: Manga) {
+        val document = Jsoup.parse(response.body().string())
+        with(map.manga) {
+            val pool = parts.get(document)
+
+            manga.author = author?.process(document, pool)
+            manga.artist = artist?.process(document, pool)
+            manga.description = summary?.process(document, pool)
+            manga.thumbnail_url = cover?.process(document, pool)
+            manga.genre = genres?.process(document, pool)
+            manga.status = status?.getStatus(document, pool) ?: Manga.UNKNOWN
+        }
+    }
+
+    override fun chapterListParse(response: Response, chapters: MutableList<Chapter>) {
+        val document = Jsoup.parse(response.body().string())
+        with(map.chapters) {
+            val pool = emptyMap<String, Element>()
+            val dateFormat = SimpleDateFormat(date?.format, Locale.ENGLISH)
+
+            for (element in document.select(chapter_css)) {
+                val chapter = Chapter.create()
+                element.select(title).first().let {
+                    chapter.name = it.text()
+                    chapter.setUrl(it.attr("href"))
+                }
+                val dateElement = element.select(date?.select).first()
+                chapter.date_upload = date?.getDate(dateElement, pool, dateFormat)?.time ?: 0
+                chapters.add(chapter)
+            }
+        }
+    }
+
+    override fun pageListParse(response: Response, pages: MutableList<Page>) {
+        val document = Jsoup.parse(response.body().string())
+        with(map.pages) {
+            val url = response.request().url().toString()
+            pages_css?.let {
+                for (element in document.select(it)) {
+                    val value = element.attr(pages_attr)
+                    val pageUrl = replace?.let { url.replace(it.toRegex(), replacement!!.replace("\$value", value)) } ?: value
+                    pages.add(Page(pages.size, pageUrl))
+                }
+            }
+
+            for ((i, element) in document.select(image_css).withIndex()) {
+                val page = pages.getOrElse(i) { Page(i, "").apply { pages.add(this) } }
+                page.imageUrl = element.attr(image_attr).let {
+                    getAbsoluteUrl(it, response.request().url())
+                }
+            }
+        }
+
+    }
+
+    override fun imageUrlParse(response: Response): String {
+        val document = Jsoup.parse(response.body().string())
+        return with(map.pages) {
+            document.select(image_css).first().attr(image_attr).let {
+                getAbsoluteUrl(it, response.request().url())
+            }
+        }
+    }
+
+}

app/src/main/java/eu/kanade/tachiyomi/data/source/base/YamlOnlineSourceMappings.kt

@@ -0,0 +1,214 @@
+@file:Suppress("UNCHECKED_CAST")
+
+package eu.kanade.tachiyomi.data.source.base
+
+import eu.kanade.tachiyomi.data.database.models.Manga
+import okhttp3.FormBody
+import okhttp3.RequestBody
+import org.jsoup.nodes.Document
+import org.jsoup.nodes.Element
+import java.text.ParseException
+import java.text.SimpleDateFormat
+import java.util.*
+
+private fun toMap(map: Any?) = map as? Map<String, Any?>
+
+class YamlSourceNode(uncheckedMap: Map<*, *>) {
+
+    val map = toMap(uncheckedMap)!!
+
+    val id: Any by map
+
+    val name: String by map
+
+    val host: String by map
+
+    val lang: String by map
+
+    val client: String?
+        get() = map["client"] as? String
+
+    val popular = PopularNode(toMap(map["popular"])!!)
+
+    val search = SearchNode(toMap(map["search"])!!)
+
+    val manga = MangaNode(toMap(map["manga"])!!)
+
+    val chapters = ChaptersNode(toMap(map["chapters"])!!)
+
+    val pages = PagesNode(toMap(map["pages"])!!)
+}
+
+interface RequestableNode {
+
+    val map: Map<String, Any?>
+
+    val url: String
+        get() = map["url"] as String
+
+    val method: String?
+        get() = map["method"] as? String
+
+    val payload: Map<String, String>?
+        get() = map["payload"] as? Map<String, String>
+
+    fun createForm(): RequestBody {
+        return FormBody.Builder().apply {
+            payload?.let {
+                for ((key, value) in it) {
+                    add(key, value)
+                }
+            }
+        }.build()
+    }
+
+}
+
+class PopularNode(override val map: Map<String, Any?>): RequestableNode {
+
+    val manga_css: String by map
+
+    val next_url_css: String?
+        get() = map["next_url_css"] as? String
+
+}
+
+class SearchNode(override val map: Map<String, Any?>): RequestableNode {
+
+    val manga_css: String by map
+
+    val next_url_css: String?
+        get() = map["next_url_css"] as? String
+}
+
+class MangaNode(private val map: Map<String, Any?>) {
+
+    val parts = CacheNode(toMap(map["parts"]) ?: emptyMap())
+
+    val artist = toMap(map["artist"])?.let { SelectableNode(it) }
+
+    val author = toMap(map["author"])?.let { SelectableNode(it) }
+
+    val summary = toMap(map["summary"])?.let { SelectableNode(it) }
+
+    val status = toMap(map["status"])?.let { StatusNode(it) }
+
+    val genres = toMap(map["genres"])?.let { SelectableNode(it) }
+
+    val cover = toMap(map["cover"])?.let { CoverNode(it) }
+
+}
+
+class ChaptersNode(private val map: Map<String, Any?>) {
+
+    val chapter_css: String by map
+
+    val title: String by map
+
+    val date = toMap(toMap(map["date"]))?.let { DateNode(it) }
+}
+
+class CacheNode(private val map: Map<String, Any?>) {
+
+    fun get(document: Document) = map.mapValues { document.select(it.value as String).first() }
+}
+
+open class SelectableNode(private val map: Map<String, Any?>) {
+
+    val select: String by map
+
+    val from: String?
+        get() = map["from"] as? String
+
+    open val attr: String?
+        get() = map["attr"] as? String
+
+    val capture: String?
+        get() = map["capture"] as? String
+
+    fun process(document: Element, cache: Map<String, Element>): String {
+        val parent = from?.let { cache[it] } ?: document
+        val node = parent.select(select).first()
+        var text = attr?.let { node.attr(it) } ?: node.text()
+        capture?.let {
+            text = Regex(it).find(text)?.groupValues?.get(1) ?: text
+        }
+        return text
+    }
+}
+
+class StatusNode(private val map: Map<String, Any?>) : SelectableNode(map) {
+
+    val complete: String?
+        get() = map["complete"] as? String
+
+    val ongoing: String?
+        get() = map["ongoing"] as? String
+
+    val licensed: String?
+        get() = map["licensed"] as? String
+
+    fun getStatus(document: Element, cache: Map<String, Element>): Int {
+        val text = process(document, cache)
+        complete?.let {
+            if (text.contains(it)) return Manga.COMPLETED
+        }
+        ongoing?.let {
+            if (text.contains(it)) return Manga.ONGOING
+        }
+        licensed?.let {
+            if (text.contains(it)) return Manga.LICENSED
+        }
+        return Manga.UNKNOWN
+    }
+}
+
+class CoverNode(private val map: Map<String, Any?>) : SelectableNode(map) {
+
+    override val attr: String?
+        get() = map["attr"] as? String ?: "src"
+}
+
+class DateNode(private val map: Map<String, Any?>) : SelectableNode(map) {
+
+    val format: String by map
+
+    fun getDate(document: Element, cache: Map<String, Element>, formatter: SimpleDateFormat): Date {
+        val text = process(document, cache)
+        try {
+            return formatter.parse(text)
+        } catch (exception: ParseException) {}
+
+        for (i in 0..7) {
+            (map["day$i"] as? List<String>)?.let {
+                it.find { it.toRegex().containsMatchIn(text) }?.let {
+                    return Calendar.getInstance().apply { add(Calendar.DATE, -i) }.time
+                }
+            }
+        }
+
+        return Date(0)
+    }
+
+}
+
+class PagesNode(private val map: Map<String, Any?>) {
+
+    val pages_css: String?
+        get() = map["pages_css"] as? String
+
+    val pages_attr: String?
+        get() = map["pages_attr"] as? String ?: "value"
+
+    val replace: String?
+        get() = map["url_replace"] as? String
+
+    val replacement: String?
+        get() = map["url_replacement"] as? String
+
+    val image_css: String by map
+
+    val image_attr: String
+        get() = map["image_attr"] as? String ?: "src"
+
+}

app/src/main/java/eu/kanade/tachiyomi/data/source/online/english/Batoto.java

@@ -1,393 +0,0 @@
-package eu.kanade.tachiyomi.data.source.online.english;
-
-import android.content.Context;
-import android.net.Uri;
-import android.text.Html;
-import android.text.TextUtils;
-
-import org.jsoup.Jsoup;
-import org.jsoup.nodes.Document;
-import org.jsoup.nodes.Element;
-import org.jsoup.select.Elements;
-
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Calendar;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import eu.kanade.tachiyomi.data.database.models.Chapter;
-import eu.kanade.tachiyomi.data.database.models.Manga;
-import eu.kanade.tachiyomi.data.network.ReqKt;
-import eu.kanade.tachiyomi.data.source.Language;
-import eu.kanade.tachiyomi.data.source.LanguageKt;
-import eu.kanade.tachiyomi.data.source.base.LoginSource;
-import eu.kanade.tachiyomi.data.source.model.MangasPage;
-import eu.kanade.tachiyomi.data.source.model.Page;
-import eu.kanade.tachiyomi.util.Parser;
-import okhttp3.Cookie;
-import okhttp3.FormBody;
-import okhttp3.Headers;
-import okhttp3.Request;
-import okhttp3.Response;
-import rx.Observable;
-import rx.functions.Func1;
-
-public class Batoto extends LoginSource {
-
-    public static final String NAME = "Batoto";
-    public static final String BASE_URL = "http://bato.to";
-    public static final String POPULAR_MANGAS_URL = BASE_URL + "/search_ajax?order_cond=views&order=desc&p=%s";
-    public static final String SEARCH_URL = BASE_URL + "/search_ajax?name=%s&p=%s";
-    public static final String CHAPTER_URL = BASE_URL + "/areader?id=%s&p=1";
-    public static final String PAGE_URL = BASE_URL + "/areader?id=%s&p=%s";
-    public static final String MANGA_URL = BASE_URL + "/comic_pop?id=%s";
-    public static final String LOGIN_URL = BASE_URL + "/forums/index.php?app=core&module=global&section=login";
-
-    public static final Pattern staffNotice = Pattern.compile("=+Batoto Staff Notice=+([^=]+)==+", Pattern.CASE_INSENSITIVE);
-
-    private final Pattern datePattern;
-    private final Map<String, Integer> dateFields;
-
-    public Batoto(Context context) {
-        super(context);
-
-        datePattern = Pattern.compile("(\\d+|A|An)\\s+(.*?)s? ago.*");
-        dateFields = new HashMap<String, Integer>() {{
-            put("second", Calendar.SECOND);
-            put("minute", Calendar.MINUTE);
-            put("hour",   Calendar.HOUR);
-            put("day",    Calendar.DATE);
-            put("week",   Calendar.WEEK_OF_YEAR);
-            put("month",  Calendar.MONTH);
-            put("year",   Calendar.YEAR);
-        }};
-    }
-
-    @Override
-    public String getName() {
-        return NAME;
-    }
-
-    @Override
-    public String getBaseUrl() {
-        return BASE_URL;
-    }
-
-    public Language getLang() {
-        return LanguageKt.getEN();
-    }
-
-    @Override
-    protected Headers.Builder headersBuilder() {
-        Headers.Builder builder = super.headersBuilder();
-        builder.add("Cookie", "lang_option=English");
-        builder.add("Referer", "http://bato.to/reader");
-        return builder;
-    }
-
-    @Override
-    public String getInitialPopularMangasUrl() {
-        return String.format(POPULAR_MANGAS_URL, 1);
-    }
-
-    @Override
-    public String getInitialSearchUrl(String query) {
-        return String.format(SEARCH_URL, Uri.encode(query), 1);
-    }
-
-    @Override
-    protected Request mangaDetailsRequest(String mangaUrl) {
-        String mangaId = mangaUrl.substring(mangaUrl.lastIndexOf('r') + 1);
-        return ReqKt.get(String.format(MANGA_URL, mangaId), getRequestHeaders());
-    }
-
-    @Override
-    protected Request pageListRequest(String pageUrl) {
-        String id = pageUrl.substring(pageUrl.indexOf('#') + 1);
-        return ReqKt.get(String.format(CHAPTER_URL, id), getRequestHeaders());
-    }
-
-    @Override
-    protected Request imageUrlRequest(Page page) {
-        String pageUrl = page.getUrl();
-        int start = pageUrl.indexOf('#') + 1;
-        int end = pageUrl.indexOf('_', start);
-        String id = pageUrl.substring(start, end);
-        return ReqKt.get(String.format(PAGE_URL, id, pageUrl.substring(end+1)), getRequestHeaders());
-    }
-
-    private List<Manga> parseMangasFromHtml(Document parsedHtml) {
-        List<Manga> mangaList = new ArrayList<>();
-
-        if (!parsedHtml.text().contains("No (more) comics found!")) {
-            for (Element currentHtmlBlock : parsedHtml.select("tr:not([id]):not([class])")) {
-                Manga manga = constructMangaFromHtmlBlock(currentHtmlBlock);
-                mangaList.add(manga);
-            }
-        }
-        return mangaList;
-    }
-
-    @Override
-    protected List<Manga> parsePopularMangasFromHtml(Document parsedHtml) {
-        return parseMangasFromHtml(parsedHtml);
-    }
-
-    @Override
-    protected String parseNextPopularMangasUrl(Document parsedHtml, MangasPage page) {
-        Element next = Parser.element(parsedHtml, "#show_more_row");
-        return next != null ? String.format(POPULAR_MANGAS_URL, page.page + 1) : null;
-    }
-
-    @Override
-    protected List<Manga> parseSearchFromHtml(Document parsedHtml) {
-        return parseMangasFromHtml(parsedHtml);
-    }
-
-    private Manga constructMangaFromHtmlBlock(Element htmlBlock) {
-        Manga manga = new Manga();
-        manga.source = getId();
-
-        Element urlElement = Parser.element(htmlBlock, "a[href^=http://bato.to]");
-        if (urlElement != null) {
-            manga.setUrl(urlElement.attr("href"));
-            manga.title = urlElement.text().trim();
-        }
-        return manga;
-    }
-
-    @Override
-    protected String parseNextSearchUrl(Document parsedHtml, MangasPage page, String query) {
-        Element next = Parser.element(parsedHtml, "#show_more_row");
-        return next != null ? String.format(SEARCH_URL, query, page.page + 1) : null;
-    }
-
-    @Override
-    protected Manga parseHtmlToManga(String mangaUrl, String unparsedHtml) {
-        Document parsedDocument = Jsoup.parse(unparsedHtml);
-
-        Element tbody = parsedDocument.select("tbody").first();
-        Element artistElement = tbody.select("tr:contains(Author/Artist:)").first();
-        Elements genreElements = tbody.select("tr:contains(Genres:) img");
-
-        Manga manga = Manga.create(mangaUrl);
-        manga.author = Parser.text(artistElement, "td:eq(1)");
-        manga.artist = Parser.text(artistElement, "td:eq(2)", manga.author);
-        manga.description = Parser.text(tbody, "tr:contains(Description:) > td:eq(1)");
-        manga.thumbnail_url = Parser.src(parsedDocument, "img[src^=http://img.bato.to/forums/uploads/]");
-        manga.status = parseStatus(Parser.text(parsedDocument, "tr:contains(Status:) > td:eq(1)"));
-
-        if (!genreElements.isEmpty()) {
-            List<String> genres = new ArrayList<>();
-            for (Element element : genreElements) {
-                genres.add(element.attr("alt"));
-            }
-            manga.genre = TextUtils.join(", ", genres);
-        }
-
-        manga.initialized = true;
-        return manga;
-    }
-
-    private int parseStatus(String status) {
-        switch (status) {
-            case "Ongoing":
-                return Manga.ONGOING;
-            case "Complete":
-                return Manga.COMPLETED;
-            default:
-                return Manga.UNKNOWN;
-        }
-    }
-
-    @Override
-    protected List<Chapter> parseHtmlToChapters(String unparsedHtml) {
-        Matcher matcher = staffNotice.matcher(unparsedHtml);
-        if (matcher.find()) {
-            String notice = Html.fromHtml(matcher.group(1)).toString().trim();
-            throw new RuntimeException(notice);
-        }
-
-        Document parsedDocument = Jsoup.parse(unparsedHtml);
-
-        List<Chapter> chapterList = new ArrayList<>();
-
-        Elements chapterElements = parsedDocument.select("tr.row.lang_English.chapter_row");
-        for (Element chapterElement : chapterElements) {
-            Chapter chapter = constructChapterFromHtmlBlock(chapterElement);
-            chapterList.add(chapter);
-        }
-        return chapterList;
-
-    }
-
-    private Chapter constructChapterFromHtmlBlock(Element chapterElement) {
-        Chapter chapter = Chapter.create();
-
-        Element urlElement = chapterElement.select("a[href^=http://bato.to/reader").first();
-        Element dateElement = chapterElement.select("td").get(4);
-
-        if (urlElement != null) {
-            String fieldUrl = urlElement.attr("href");
-            chapter.setUrl(fieldUrl);
-            chapter.name = urlElement.text().trim();
-        }
-        if (dateElement != null) {
-            chapter.date_upload = parseDateFromElement(dateElement);
-        }
-        return chapter;
-    }
-
-    @SuppressWarnings("WrongConstant")
-    private long parseDateFromElement(Element dateElement) {
-        String dateAsString = dateElement.text();
-
-        Date date;
-        try {
-            date = new SimpleDateFormat("dd MMMMM yyyy - hh:mm a", Locale.ENGLISH).parse(dateAsString);
-        } catch (ParseException e) {
-            Matcher m = datePattern.matcher(dateAsString);
-
-            if (m.matches()) {
-                String number = m.group(1);
-                int amount = number.contains("A") ? 1 : Integer.parseInt(m.group(1));
-                String unit = m.group(2);
-
-                Calendar cal = Calendar.getInstance();
-                cal.add(dateFields.get(unit), -amount);
-                date = cal.getTime();
-            } else {
-                return 0;
-            }
-        }
-        return date.getTime();
-    }
-
-    @Override
-    protected List<String> parseHtmlToPageUrls(String unparsedHtml) {
-        Document parsedDocument = Jsoup.parse(unparsedHtml);
-
-        List<String> pageUrlList = new ArrayList<>();
-
-        Element selectElement = Parser.element(parsedDocument, "#page_select");
-        if (selectElement != null) {
-            for (Element pageUrlElement : selectElement.select("option")) {
-                pageUrlList.add(pageUrlElement.attr("value"));
-            }
-        } else {
-            // For webtoons in one page
-            for (int i = 0; i < parsedDocument.select("div > img").size(); i++) {
-                pageUrlList.add("");
-            }
-        }
-
-        return pageUrlList;
-    }
-
-    @Override
-    protected List<Page> parseFirstPage(List<? extends Page> pages, String unparsedHtml) {
-        if (!unparsedHtml.contains("Want to see this chapter per page instead?")) {
-            String firstImage = parseHtmlToImageUrl(unparsedHtml);
-            pages.get(0).setImageUrl(firstImage);
-        } else {
-            // For webtoons in one page
-            Document parsedDocument = Jsoup.parse(unparsedHtml);
-            Elements imageUrls = parsedDocument.select("div > img");
-            for (int i = 0; i < pages.size(); i++) {
-                pages.get(i).setImageUrl(imageUrls.get(i).attr("src"));
-            }
-        }
-        return (List<Page>) pages;
-    }
-
-    @Override
-    protected String parseHtmlToImageUrl(String unparsedHtml) {
-        int beginIndex = unparsedHtml.indexOf("<img id=\"comic_page\"");
-        int endIndex = unparsedHtml.indexOf("</a>", beginIndex);
-        String trimmedHtml = unparsedHtml.substring(beginIndex, endIndex);
-
-        Document parsedDocument = Jsoup.parse(trimmedHtml);
-        Element imageElement = parsedDocument.getElementById("comic_page");
-        return imageElement.attr("src");
-    }
-
-    @Override
-    public Observable<Boolean> login(final String username, final String password) {
-        return getNetworkService().requestBody(ReqKt.get(LOGIN_URL, getRequestHeaders()))
-                .flatMap(new Func1<String, Observable<Response>>() {
-                    @Override
-                    public Observable<Response> call(String response) {return doLogin(response, username, password);}
-                })
-                .map(new Func1<Response, Boolean>() {
-                    @Override
-                    public Boolean call(Response resp) {return isAuthenticationSuccessful(resp);}
-                });
-    }
-
-    private Observable<Response> doLogin(String response, String username, String password) {
-        Document doc = Jsoup.parse(response);
-        Element form = doc.select("#login").first();
-        String postUrl = form.attr("action");
-
-        FormBody.Builder formBody = new FormBody.Builder();
-        Element authKey = form.select("input[name=auth_key]").first();
-
-        formBody.add(authKey.attr("name"), authKey.attr("value"));
-        formBody.add("ips_username", username);
-        formBody.add("ips_password", password);
-        formBody.add("invisible", "1");
-        formBody.add("rememberMe", "1");
-
-        return getNetworkService().request(ReqKt.post(postUrl, getRequestHeaders(), formBody.build()));
-    }
-
-    @Override
-    protected boolean isAuthenticationSuccessful(Response response) {
-        return response.priorResponse() != null && response.priorResponse().code() == 302;
-    }
-
-    @Override
-    public boolean isLogged() {
-        try {
-            for (Cookie cookie : getNetworkService().getCookies().get(new URI(BASE_URL))) {
-                if (cookie.name().equals("pass_hash"))
-                    return true;
-            }
-
-        } catch (URISyntaxException e) {
-            e.printStackTrace();
-        }
-        return false;
-    }
-
-    @Override
-    public Observable<List<Chapter>> pullChaptersFromNetwork(final String mangaUrl) {
-        Observable<List<Chapter>> observable;
-        String username = getPrefs().sourceUsername(this);
-        String password = getPrefs().sourcePassword(this);
-        if (username.isEmpty() && password.isEmpty()) {
-            observable = Observable.error(new Exception("User not logged"));
-        }
-        else if (!isLogged()) {
-            observable = login(username, password)
-                    .flatMap(new Func1<Boolean, Observable<? extends List<Chapter>>>() {
-                        @Override
-                        public Observable<? extends List<Chapter>> call(Boolean result) {return Batoto.super.pullChaptersFromNetwork(mangaUrl);}
-                    });
-        }
-        else {
-            observable = super.pullChaptersFromNetwork(mangaUrl);
-        }
-        return observable;
-    }
-
-}

app/src/main/java/eu/kanade/tachiyomi/data/source/online/english/Batoto.kt

@@ -0,0 +1,270 @@
+package eu.kanade.tachiyomi.data.source.online.english
+
+import android.content.Context
+import android.net.Uri
+import android.text.Html
+import eu.kanade.tachiyomi.data.database.models.Chapter
+import eu.kanade.tachiyomi.data.database.models.Manga
+import eu.kanade.tachiyomi.data.network.get
+import eu.kanade.tachiyomi.data.network.post
+import eu.kanade.tachiyomi.data.source.EN
+import eu.kanade.tachiyomi.data.source.Language
+import eu.kanade.tachiyomi.data.source.base.ParsedOnlineSource
+import eu.kanade.tachiyomi.data.source.model.MangasPage
+import eu.kanade.tachiyomi.data.source.model.Page
+import okhttp3.FormBody
+import okhttp3.Request
+import okhttp3.Response
+import org.jsoup.Jsoup
+import org.jsoup.nodes.Document
+import org.jsoup.nodes.Element
+import rx.Observable
+import java.net.URI
+import java.net.URISyntaxException
+import java.text.ParseException
+import java.text.SimpleDateFormat
+import java.util.*
+import java.util.regex.Pattern
+
+class Batoto(context: Context, override val id: Int) : ParsedOnlineSource(context) {
+
+    override val name = "Batoto"
+
+    override val baseUrl = "http://bato.to"
+
+    override val lang: Language get() = EN
+
+    private val datePattern = Pattern.compile("(\\d+|A|An)\\s+(.*?)s? ago.*")
+
+    private val dateFields = HashMap<String, Int>().apply {
+        put("second", Calendar.SECOND)
+        put("minute", Calendar.MINUTE)
+        put("hour", Calendar.HOUR)
+        put("day", Calendar.DATE)
+        put("week", Calendar.WEEK_OF_YEAR)
+        put("month", Calendar.MONTH)
+        put("year", Calendar.YEAR)
+    }
+
+    private val staffNotice = Pattern.compile("=+Batoto Staff Notice=+([^=]+)==+", Pattern.CASE_INSENSITIVE)
+
+    override fun headersBuilder() = super.headersBuilder()
+            .add("Cookie", "lang_option=English")
+            .add("Referer", "http://bato.to/reader")
+
+    override fun popularMangaInitialUrl() = "$baseUrl/search_ajax?order_cond=views&order=desc&p=1"
+
+    override fun searchMangaInitialUrl(query: String) = "$baseUrl/search_ajax?name=${Uri.encode(query)}&p=1"
+
+    override fun mangaDetailsRequest(manga: Manga): Request {
+        val mangaId = manga.url.substringAfterLast("r")
+        return get("$baseUrl/comic_pop?id=$mangaId", headers)
+    }
+
+    override fun pageListRequest(chapter: Chapter): Request {
+        val id = chapter.url.substringAfterLast("#")
+        return get("$baseUrl/areader?id=$id&p=1", headers)
+    }
+
+    override fun imageUrlRequest(page: Page): Request {
+        val pageUrl = page.url
+        val start = pageUrl.indexOf("#") + 1
+        val end = pageUrl.indexOf("_", start)
+        val id = pageUrl.substring(start, end)
+        return get("$baseUrl/areader?id=$id&p=${pageUrl.substring(end+1)}", headers)
+    }
+
+    override fun popularMangaParse(response: Response, page: MangasPage) {
+        val document = Jsoup.parse(response.body().string())
+        for (element in document.select(popularMangaSelector())) {
+            Manga().apply {
+                source = [email protected]
+                popularMangaFromElement(element, this)
+                page.mangas.add(this)
+            }
+        }
+
+        page.nextPageUrl = document.select(popularMangaNextPageSelector()).first()?.let {
+            "$baseUrl/search_ajax?order_cond=views&order=desc&p=${page.page + 1}"
+        }
+    }
+
+    override fun popularMangaSelector() = "tr:not([id]):not([class])"
+
+    override fun popularMangaFromElement(element: Element, manga: Manga) {
+        element.select("a[href^=http://bato.to]").first().let {
+            manga.setUrl(it.attr("href"))
+            manga.title = it.text().trim()
+        }
+    }
+
+    override fun popularMangaNextPageSelector() = "#show_more_row"
+
+    override fun searchMangaParse(response: Response, page: MangasPage, query: String) {
+        val document = Jsoup.parse(response.body().string())
+        for (element in document.select(searchMangaSelector())) {
+            Manga().apply {
+                source = [email protected]
+                searchMangaFromElement(element, this)
+                page.mangas.add(this)
+            }
+        }
+
+        page.nextPageUrl = document.select(searchMangaNextPageSelector()).first()?.let {
+            "$baseUrl/search_ajax?name=${Uri.encode(query)}&p=${page.page + 1}"
+        }
+    }
+
+    override fun searchMangaSelector() = popularMangaSelector()
+
+    override fun searchMangaFromElement(element: Element, manga: Manga) {
+        popularMangaFromElement(element, manga)
+    }
+
+    override fun searchMangaNextPageSelector() = popularMangaNextPageSelector()
+
+    override fun mangaDetailsParse(document: Document, manga: Manga) {
+        val tbody = document.select("tbody").first()
+        val artistElement = tbody.select("tr:contains(Author/Artist:)").first()
+
+        manga.author = artistElement.select("td:eq(1)").first()?.text()
+        manga.artist = artistElement.select("td:eq(2)").first()?.text() ?: manga.author
+        manga.description = tbody.select("tr:contains(Description:) > td:eq(1)").first()?.text()
+        manga.thumbnail_url = document.select("img[src^=http://img.bato.to/forums/uploads/]").first()?.attr("src")
+        manga.status = parseStatus(document.select("tr:contains(Status:) > td:eq(1)").first()?.text())
+        manga.genre = tbody.select("tr:contains(Genres:) img").map { it.attr("alt") }.joinToString(", ")
+    }
+
+    private fun parseStatus(status: String?) = when (status) {
+        "Ongoing" -> Manga.ONGOING
+        "Complete" -> Manga.COMPLETED
+        else -> Manga.UNKNOWN
+    }
+
+    override fun chapterListParse(response: Response, chapters: MutableList<Chapter>) {
+        val body = response.body().string()
+        val matcher = staffNotice.matcher(body)
+        if (matcher.find()) {
+            val notice = Html.fromHtml(matcher.group(1)).toString().trim()
+            throw RuntimeException(notice)
+        }
+
+        val document = Jsoup.parse(body)
+
+        for (element in document.select(chapterListSelector())) {
+            Chapter.create().apply {
+                chapterFromElement(element, this)
+                chapters.add(this)
+            }
+        }
+    }
+
+    override fun chapterListSelector() = "tr.row.lang_English.chapter_row"
+
+    override fun chapterFromElement(element: Element, chapter: Chapter) {
+        val urlElement = element.select("a[href^=http://bato.to/reader").first()
+
+        chapter.setUrl(urlElement.attr("href"))
+        chapter.name = urlElement.text()
+        chapter.date_upload = element.select("td").getOrNull(4)?.let {
+            parseDateFromElement(it)
+        } ?: 0
+    }
+
+    private fun parseDateFromElement(dateElement: Element): Long {
+        val dateAsString = dateElement.text()
+
+        val date: Date
+        try {
+            date = SimpleDateFormat("dd MMMMM yyyy - hh:mm a", Locale.ENGLISH).parse(dateAsString)
+        } catch (e: ParseException) {
+            val m = datePattern.matcher(dateAsString)
+
+            if (m.matches()) {
+                val number = m.group(1)
+                val amount = if (number.contains("A")) 1 else Integer.parseInt(m.group(1))
+                val unit = m.group(2)
+
+                date = Calendar.getInstance().apply {
+                    add(dateFields[unit]!!, -amount)
+                }.time
+            } else {
+                return 0
+            }
+        }
+
+        return date.time
+    }
+
+    override fun pageListParse(document: Document, pages: MutableList<Page>) {
+        val selectElement = document.select("#page_select").first()
+        if (selectElement != null) {
+            for ((i, element) in selectElement.select("option").withIndex()) {
+                pages.add(Page(i, element.attr("value")))
+            }
+        } else {
+            // For webtoons in one page
+            for ((i, element) in document.select("div > img").withIndex()) {
+                pages.add(Page(i, "", element.attr("src")))
+            }
+        }
+    }
+
+    override fun imageUrlParse(document: Document): String {
+        return document.select("#comic_page").first().attr("src")
+    }
+
+    override fun login(username: String, password: String) =
+        network.request(get("$baseUrl/forums/index.php?app=core&module=global&section=login", headers))
+                .map { it.body().string() }
+                .flatMap { doLogin(it, username, password) }
+                .map { isAuthenticationSuccessful(it) }
+
+    private fun doLogin(response: String, username: String, password: String): Observable<Response> {
+        val doc = Jsoup.parse(response)
+        val form = doc.select("#login").first()
+        val url = form.attr("action")
+        val authKey = form.select("input[name=auth_key]").first()
+
+        val payload = FormBody.Builder().apply {
+            add(authKey.attr("name"), authKey.attr("value"))
+            add("ips_username", username)
+            add("ips_password", password)
+            add("invisible", "1")
+            add("rememberMe", "1")
+        }.build()
+
+        return network.request(post(url, headers, payload))
+    }
+
+    override fun isLoginRequired() = true
+
+    override fun isAuthenticationSuccessful(response: Response) =
+        response.priorResponse() != null && response.priorResponse().code() == 302
+
+    override fun isLogged(): Boolean {
+        try {
+            return network.cookies.get(URI(baseUrl)).find { it.name() == "pass_hash" } != null
+        } catch (e: URISyntaxException) {
+            // Ignore
+        }
+        return false
+    }
+
+    override fun fetchChapterList(manga: Manga): Observable<List<Chapter>> {
+        if (!isLogged()) {
+            val username = preferences.sourceUsername(this)
+            val password = preferences.sourcePassword(this)
+
+            if (username.isNullOrEmpty() || password.isNullOrEmpty()) {
+                return Observable.error(Exception("User not logged"))
+            } else {
+                return login(username, password).flatMap { super.fetchChapterList(manga) }
+            }
+
+        } else {
+            return super.fetchChapterList(manga)
+        }
+    }
+
+}

app/src/main/java/eu/kanade/tachiyomi/data/source/online/english/Kissmanga.kt

@@ -6,195 +6,116 @@ import eu.kanade.tachiyomi.data.database.models.Manga
 import eu.kanade.tachiyomi.data.network.get
 import eu.kanade.tachiyomi.data.network.post
 import eu.kanade.tachiyomi.data.source.EN
-import eu.kanade.tachiyomi.data.source.base.Source
+import eu.kanade.tachiyomi.data.source.Language
+import eu.kanade.tachiyomi.data.source.base.ParsedOnlineSource
 import eu.kanade.tachiyomi.data.source.model.MangasPage
 import eu.kanade.tachiyomi.data.source.model.Page
-import eu.kanade.tachiyomi.util.Parser
 import okhttp3.FormBody
 import okhttp3.OkHttpClient
 import okhttp3.Request
-import org.jsoup.Jsoup
+import okhttp3.Response
 import org.jsoup.nodes.Document
 import org.jsoup.nodes.Element
-import java.text.ParseException
 import java.text.SimpleDateFormat
-import java.util.*
 import java.util.regex.Pattern
 
-class Kissmanga(context: Context) : Source(context) {
+class Kissmanga(context: Context, override val id: Int) : ParsedOnlineSource(context) {
 
-    override fun getName() = NAME
+    override val name = "Kissmanga"
 
-    override fun getBaseUrl() = BASE_URL
+    override val baseUrl = "http://kissmanga.com"
 
-    override fun getLang() = EN
+    override val lang: Language get() = EN
 
-    override val networkClient: OkHttpClient
-        get() = networkService.cloudflareClient
+    override val client: OkHttpClient get() = network.cloudflareClient
 
-    override fun getInitialPopularMangasUrl(): String {
-        return String.format(POPULAR_MANGAS_URL, 1)
-    }
+    override fun popularMangaInitialUrl() = "$baseUrl/MangaList/MostPopular"
 
-    override fun getInitialSearchUrl(query: String): String {
-        return SEARCH_URL
-    }
+    override fun popularMangaSelector() = "table.listing tr:gt(1)"
 
-    override fun searchMangaRequest(page: MangasPage, query: String): Request {
-        if (page.page == 1) {
-            page.url = getInitialSearchUrl(query)
+    override fun popularMangaFromElement(element: Element, manga: Manga) {
+        element.select("td a:eq(0)").first().let {
+            manga.setUrl(it.attr("href"))
+            manga.title = it.text()
         }
-
-        val form = FormBody.Builder()
-        form.add("authorArtist", "")
-        form.add("mangaName", query)
-        form.add("status", "")
-        form.add("genres", "")
-
-        return post(page.url, requestHeaders, form.build())
-    }
-
-    override fun pageListRequest(chapterUrl: String): Request {
-        return post(baseUrl + chapterUrl, requestHeaders)
-    }
-
-    override fun imageRequest(page: Page): Request {
-        return get(page.imageUrl)
     }
 
-    override fun parsePopularMangasFromHtml(parsedHtml: Document): List<Manga> {
-        val mangaList = ArrayList<Manga>()
-
-        for (currentHtmlBlock in parsedHtml.select("table.listing tr:gt(1)")) {
-            val manga = constructPopularMangaFromHtml(currentHtmlBlock)
-            mangaList.add(manga)
+    override fun searchMangaRequest(page: MangasPage, query: String): Request {
+        if (page.page == 1) {
+            page.url = searchMangaInitialUrl(query)
         }
 
-        return mangaList
-    }
-
-    private fun constructPopularMangaFromHtml(htmlBlock: Element): Manga {
-        val manga = Manga()
-        manga.source = id
+        val form = FormBody.Builder().apply {
+            add("authorArtist", "")
+            add("mangaName", query)
+            add("status", "")
+            add("genres", "")
+        }.build()
 
-        val urlElement = Parser.element(htmlBlock, "td a:eq(0)")
-
-        if (urlElement != null) {
-            manga.setUrl(urlElement.attr("href"))
-            manga.title = urlElement.text()
-        }
-
-        return manga
+        return post(page.url, headers, form)
     }
 
-    override fun parseNextPopularMangasUrl(parsedHtml: Document, page: MangasPage): String? {
-        val path = Parser.href(parsedHtml, "li > a:contains(› Next)")
-        return if (path != null) BASE_URL + path else null
-    }
+    override fun popularMangaNextPageSelector() = "li > a:contains(› Next)"
 
-    override fun parseSearchFromHtml(parsedHtml: Document): List<Manga> {
-        return parsePopularMangasFromHtml(parsedHtml)
-    }
+    override fun searchMangaSelector() = popularMangaSelector()
 
-    override fun parseNextSearchUrl(parsedHtml: Document, page: MangasPage, query: String): String? {
-        return null
+    override fun searchMangaFromElement(element: Element, manga: Manga) {
+        popularMangaFromElement(element, manga)
     }
 
-    override fun parseHtmlToManga(mangaUrl: String, unparsedHtml: String): Manga {
-        val parsedDocument = Jsoup.parse(unparsedHtml)
-        val infoElement = parsedDocument.select("div.barContent").first()
+    override fun searchMangaNextPageSelector() = null
 
-        val manga = Manga.create(mangaUrl)
-        manga.title = Parser.text(infoElement, "a.bigChar")
-        manga.author = Parser.text(infoElement, "p:has(span:contains(Author:)) > a")
-        manga.genre = Parser.allText(infoElement, "p:has(span:contains(Genres:)) > *:gt(0)")
-        manga.description = Parser.allText(infoElement, "p:has(span:contains(Summary:)) ~ p")
-        manga.status = parseStatus(Parser.text(infoElement, "p:has(span:contains(Status:))")!!)
+    override fun searchMangaInitialUrl(query: String) = "$baseUrl/AdvanceSearch"
 
-        val thumbnail = Parser.src(parsedDocument, ".rightBox:eq(0) img")
-        if (thumbnail != null) {
-            manga.thumbnail_url = thumbnail
-        }
+    override fun mangaDetailsParse(document: Document, manga: Manga) {
+        val infoElement = document.select("div.barContent").first()
 
-        manga.initialized = true
-        return manga
+        manga.author = infoElement.select("p:has(span:contains(Author:)) > a").first()?.text()
+        manga.genre = infoElement.select("p:has(span:contains(Genres:)) > *:gt(0)").text()
+        manga.description = infoElement.select("p:has(span:contains(Summary:)) ~ p").text()
+        manga.status = parseStatus(infoElement.select("p:has(span:contains(Status:))").first()?.text())
+        manga.thumbnail_url = document.select(".rightBox:eq(0) img").first()?.attr("src")
     }
 
-    private fun parseStatus(status: String): Int {
-        if (status.contains("Ongoing")) {
-            return Manga.ONGOING
-        }
-        if (status.contains("Completed")) {
-            return Manga.COMPLETED
+    fun parseStatus(status: String?): Int {
+        if (status != null) {
+            when {
+                status.contains("Ongoing") -> return Manga.ONGOING
+                status.contains("Completed") -> return Manga.COMPLETED
+            }
         }
         return Manga.UNKNOWN
     }
 
-    override fun parseHtmlToChapters(unparsedHtml: String): List<Chapter> {
-        val parsedDocument = Jsoup.parse(unparsedHtml)
-        val chapterList = ArrayList<Chapter>()
+    override fun chapterListSelector() = "table.listing tr:gt(1)"
 
-        for (chapterElement in parsedDocument.select("table.listing tr:gt(1)")) {
-            val chapter = constructChapterFromHtmlBlock(chapterElement)
-            chapterList.add(chapter)
-        }
+    override fun chapterFromElement(element: Element, chapter: Chapter) {
+        val urlElement = element.select("a").first()
 
-        return chapterList
+        chapter.setUrl(urlElement.attr("href"))
+        chapter.name = urlElement.text()
+        chapter.date_upload = element.select("td:eq(1)").first()?.text()?.let {
+            SimpleDateFormat("MM/dd/yyyy").parse(it).time
+        } ?: 0
     }
 
-    private fun constructChapterFromHtmlBlock(chapterElement: Element): Chapter {
-        val chapter = Chapter.create()
+    override fun pageListRequest(chapter: Chapter) = post(baseUrl + chapter.url, headers)
 
-        val urlElement = Parser.element(chapterElement, "a")
-        val date = Parser.text(chapterElement, "td:eq(1)")
-
-        if (urlElement != null) {
-            chapter.setUrl(urlElement.attr("href"))
-            chapter.name = urlElement.text()
-        }
-        if (date != null) {
-            try {
-                chapter.date_upload = SimpleDateFormat("MM/dd/yyyy", Locale.ENGLISH).parse(date).time
-            } catch (e: ParseException) { /* Ignore */
-            }
-
-        }
-        return chapter
-    }
-
-    override fun parseHtmlToPageUrls(unparsedHtml: String): List<String> {
-        val parsedDocument = Jsoup.parse(unparsedHtml)
-        val pageUrlList = ArrayList<String>()
-
-        val numImages = parsedDocument.select("#divImage img").size
-
-        for (i in 0..numImages - 1) {
-            pageUrlList.add("")
-        }
-        return pageUrlList
-    }
-
-    override fun parseFirstPage(pages: List<Page>, unparsedHtml: String): List<Page> {
+    override fun pageListParse(response: Response, pages: MutableList<Page>) {
         val p = Pattern.compile("lstImages.push\\(\"(.+?)\"")
-        val m = p.matcher(unparsedHtml)
+        val m = p.matcher(response.body().string())
 
         var i = 0
         while (m.find()) {
-            pages[i++].imageUrl = m.group(1)
+            pages.add(Page(i++, "", m.group(1)))
         }
-        return pages
     }
 
-    override fun parseHtmlToImageUrl(unparsedHtml: String): String? {
-        return null
-    }
+    // Not used
+    override fun pageListParse(document: Document, pages: MutableList<Page>) {}
 
-    companion object {
+    override fun imageUrlRequest(page: Page) = get(page.url)
 
-        val NAME = "Kissmanga"
-        val BASE_URL = "http://kissmanga.com"
-        val POPULAR_MANGAS_URL = BASE_URL + "/MangaList/MostPopular?page=%s"
-        val SEARCH_URL = BASE_URL + "/AdvanceSearch"
-    }
+    override fun imageUrlParse(document: Document) = ""
 
-}
+}

app/src/main/java/eu/kanade/tachiyomi/data/source/online/english/Mangafox.java

@@ -1,245 +0,0 @@
-package eu.kanade.tachiyomi.data.source.online.english;
-
-import android.content.Context;
-import android.net.Uri;
-
-import org.jsoup.Jsoup;
-import org.jsoup.nodes.Document;
-import org.jsoup.nodes.Element;
-import org.jsoup.select.Elements;
-
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Calendar;
-import java.util.Date;
-import java.util.List;
-import java.util.Locale;
-
-import eu.kanade.tachiyomi.data.database.models.Chapter;
-import eu.kanade.tachiyomi.data.database.models.Manga;
-import eu.kanade.tachiyomi.data.source.Language;
-import eu.kanade.tachiyomi.data.source.LanguageKt;
-import eu.kanade.tachiyomi.data.source.base.Source;
-import eu.kanade.tachiyomi.data.source.model.MangasPage;
-import eu.kanade.tachiyomi.util.Parser;
-
-public class Mangafox extends Source {
-
-    public static final String NAME = "Mangafox";
-    public static final String BASE_URL = "http://mangafox.me";
-    public static final String POPULAR_MANGAS_URL = BASE_URL + "/directory/%s";
-    public static final String SEARCH_URL =
-            BASE_URL + "/search.php?name_method=cw&advopts=1&order=za&sort=views&name=%s&page=%s";
-
-    public Mangafox(Context context) {
-        super(context);
-    }
-
-    @Override
-    public String getName() {
-        return NAME;
-    }
-
-    @Override
-    public String getBaseUrl() {
-        return BASE_URL;
-    }
-
-    public Language getLang() {
-        return LanguageKt.getEN();
-    }
-
-    @Override
-    protected String getInitialPopularMangasUrl() {
-        return String.format(POPULAR_MANGAS_URL, "");
-    }
-
-    @Override
-    protected String getInitialSearchUrl(String query) {
-        return String.format(SEARCH_URL, Uri.encode(query), 1);
-    }
-
-    @Override
-    protected List<Manga> parsePopularMangasFromHtml(Document parsedHtml) {
-        List<Manga> mangaList = new ArrayList<>();
-
-        for (Element currentHtmlBlock : parsedHtml.select("div#mangalist > ul.list > li")) {
-            Manga currentManga = constructPopularMangaFromHtmlBlock(currentHtmlBlock);
-            mangaList.add(currentManga);
-        }
-        return mangaList;
-    }
-
-    private Manga constructPopularMangaFromHtmlBlock(Element htmlBlock) {
-        Manga manga = new Manga();
-        manga.source = getId();
-
-        Element urlElement = Parser.element(htmlBlock, "a.title");
-        if (urlElement != null) {
-            manga.setUrl(urlElement.attr("href"));
-            manga.title = urlElement.text();
-        }
-        return manga;
-    }
-
-    @Override
-    protected String parseNextPopularMangasUrl(Document parsedHtml, MangasPage page) {
-        Element next = Parser.element(parsedHtml, "a:has(span.next)");
-        return next != null ? String.format(POPULAR_MANGAS_URL, next.attr("href")) : null;
-    }
-
-    @Override
-    protected List<Manga> parseSearchFromHtml(Document parsedHtml) {
-        List<Manga> mangaList = new ArrayList<>();
-
-        for (Element currentHtmlBlock : parsedHtml.select("table#listing > tbody > tr:gt(0)")) {
-            Manga currentManga = constructSearchMangaFromHtmlBlock(currentHtmlBlock);
-            mangaList.add(currentManga);
-        }
-        return mangaList;
-    }
-
-    private Manga constructSearchMangaFromHtmlBlock(Element htmlBlock) {
-        Manga mangaFromHtmlBlock = new Manga();
-        mangaFromHtmlBlock.source = getId();
-
-        Element urlElement = Parser.element(htmlBlock, "a.series_preview");
-        if (urlElement != null) {
-            mangaFromHtmlBlock.setUrl(urlElement.attr("href"));
-            mangaFromHtmlBlock.title = urlElement.text();
-        }
-        return mangaFromHtmlBlock;
-    }
-
-    @Override
-    protected String parseNextSearchUrl(Document parsedHtml, MangasPage page, String query) {
-        Element next = Parser.element(parsedHtml, "a:has(span.next)");
-        return next != null ? BASE_URL + next.attr("href") : null;
-    }
-
-    @Override
-    protected Manga parseHtmlToManga(String mangaUrl, String unparsedHtml) {
-        Document parsedDocument = Jsoup.parse(unparsedHtml);
-
-        Element infoElement = parsedDocument.select("div#title").first();
-        Element rowElement = infoElement.select("table > tbody > tr:eq(1)").first();
-        Element sideInfoElement = parsedDocument.select("#series_info").first();
-
-        Manga manga = Manga.create(mangaUrl);
-        manga.author = Parser.text(rowElement, "td:eq(1)");
-        manga.artist = Parser.text(rowElement, "td:eq(2)");
-        manga.description = Parser.text(infoElement, "p.summary");
-        manga.genre = Parser.text(rowElement, "td:eq(3)");
-        manga.thumbnail_url = Parser.src(sideInfoElement, "div.cover > img");
-        manga.status = parseStatus(Parser.text(sideInfoElement, ".data"));
-
-        manga.initialized = true;
-        return manga;
-    }
-
-    private int parseStatus(String status) {
-        if (status.contains("Ongoing")) {
-            return Manga.ONGOING;
-        }
-        if (status.contains("Completed")) {
-            return Manga.COMPLETED;
-        }
-        return Manga.UNKNOWN;
-    }
-
-    @Override
-    protected List<Chapter> parseHtmlToChapters(String unparsedHtml) {
-        Document parsedDocument = Jsoup.parse(unparsedHtml);
-
-        List<Chapter> chapterList = new ArrayList<>();
-
-        for (Element chapterElement : parsedDocument.select("div#chapters li div")) {
-            Chapter currentChapter = constructChapterFromHtmlBlock(chapterElement);
-            chapterList.add(currentChapter);
-        }
-        return chapterList;
-    }
-
-    private Chapter constructChapterFromHtmlBlock(Element chapterElement) {
-        Chapter chapter = Chapter.create();
-
-        Element urlElement = chapterElement.select("a.tips").first();
-        Element dateElement = chapterElement.select("span.date").first();
-
-        if (urlElement != null) {
-            chapter.setUrl(urlElement.attr("href"));
-            chapter.name = urlElement.text();
-        }
-        if (dateElement != null) {
-            chapter.date_upload = parseUpdateFromElement(dateElement);
-        }
-        return chapter;
-    }
-
-    private long parseUpdateFromElement(Element updateElement) {
-        String updatedDateAsString = updateElement.text();
-
-        if (updatedDateAsString.contains("Today")) {
-            Calendar today = Calendar.getInstance();
-            today.set(Calendar.HOUR_OF_DAY, 0);
-            today.set(Calendar.MINUTE, 0);
-            today.set(Calendar.SECOND, 0);
-            today.set(Calendar.MILLISECOND, 0);
-
-            try {
-                Date withoutDay = new SimpleDateFormat("h:mm a", Locale.ENGLISH).parse(updatedDateAsString.replace("Today", ""));
-                return today.getTimeInMillis() + withoutDay.getTime();
-            } catch (ParseException e) {
-                return today.getTimeInMillis();
-            }
-        } else if (updatedDateAsString.contains("Yesterday")) {
-            Calendar yesterday = Calendar.getInstance();
-            yesterday.add(Calendar.DATE, -1);
-            yesterday.set(Calendar.HOUR_OF_DAY, 0);
-            yesterday.set(Calendar.MINUTE, 0);
-            yesterday.set(Calendar.SECOND, 0);
-            yesterday.set(Calendar.MILLISECOND, 0);
-
-            try {
-                Date withoutDay = new SimpleDateFormat("h:mm a", Locale.ENGLISH).parse(updatedDateAsString.replace("Yesterday", ""));
-                return yesterday.getTimeInMillis() + withoutDay.getTime();
-            } catch (ParseException e) {
-                return yesterday.getTimeInMillis();
-            }
-        } else {
-            try {
-                Date specificDate = new SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH).parse(updatedDateAsString);
-
-                return specificDate.getTime();
-            } catch (ParseException e) {
-                // Do Nothing.
-            }
-        }
-
-        return 0;
-    }
-
-    @Override
-    protected List<String> parseHtmlToPageUrls(String unparsedHtml) {
-        Document parsedDocument = Jsoup.parse(unparsedHtml);
-
-        List<String> pageUrlList = new ArrayList<>();
-
-        Elements pageUrlElements = parsedDocument.select("select.m").first().select("option:not([value=0])");
-        String baseUrl = parsedDocument.select("div#series a").first().attr("href").replace("1.html", "");
-        for (Element pageUrlElement : pageUrlElements) {
-            pageUrlList.add(baseUrl + pageUrlElement.attr("value") + ".html");
-        }
-
-        return pageUrlList;
-    }
-
-    @Override
-    protected String parseHtmlToImageUrl(String unparsedHtml) {
-        Document parsedDocument = Jsoup.parse(unparsedHtml);
-
-        Element imageElement = parsedDocument.getElementById("image");
-        return imageElement.attr("src");
-    }
-}

app/src/main/java/eu/kanade/tachiyomi/data/source/online/english/Mangahere.java

@@ -1,313 +0,0 @@
-package eu.kanade.tachiyomi.data.source.online.english;
-
-import android.content.Context;
-import android.net.Uri;
-
-import org.jsoup.Jsoup;
-import org.jsoup.nodes.Document;
-import org.jsoup.nodes.Element;
-import org.jsoup.select.Elements;
-
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Calendar;
-import java.util.Date;
-import java.util.List;
-import java.util.Locale;
-
-import eu.kanade.tachiyomi.data.database.models.Chapter;
-import eu.kanade.tachiyomi.data.database.models.Manga;
-import eu.kanade.tachiyomi.data.source.Language;
-import eu.kanade.tachiyomi.data.source.LanguageKt;
-import eu.kanade.tachiyomi.data.source.base.Source;
-import eu.kanade.tachiyomi.data.source.model.MangasPage;
-import eu.kanade.tachiyomi.util.Parser;
-
-public class Mangahere extends Source {
-
-    public static final String NAME = "Mangahere";
-    public static final String BASE_URL = "http://www.mangahere.co";
-    public static final String POPULAR_MANGAS_URL = BASE_URL + "/directory/%s";
-    public static final String SEARCH_URL = BASE_URL + "/search.php?name=%s&page=%s&sort=views&order=za";
-
-    public Mangahere(Context context) {
-        super(context);
-    }
-
-    @Override
-    public String getName() {
-        return NAME;
-    }
-
-    @Override
-    public String getBaseUrl() {
-        return BASE_URL;
-    }
-
-    public Language getLang() {
-        return LanguageKt.getEN();
-    }
-
-    @Override
-    protected String getInitialPopularMangasUrl() {
-        return String.format(POPULAR_MANGAS_URL, "");
-    }
-
-    @Override
-    protected String getInitialSearchUrl(String query) {
-        return String.format(SEARCH_URL, Uri.encode(query), 1);
-    }
-
-    @Override
-    public List<Manga> parsePopularMangasFromHtml(Document parsedHtml) {
-        List<Manga> mangaList = new ArrayList<>();
-
-        for (Element currentHtmlBlock : parsedHtml.select("div.directory_list > ul > li")) {
-            Manga currentManga = constructPopularMangaFromHtmlBlock(currentHtmlBlock);
-            mangaList.add(currentManga);
-        }
-        return mangaList;
-    }
-
-    private Manga constructPopularMangaFromHtmlBlock(Element htmlBlock) {
-        Manga manga = new Manga();
-        manga.source = getId();
-
-        Element urlElement = Parser.element(htmlBlock, "div.title > a");
-        if (urlElement != null) {
-            manga.setUrl(urlElement.attr("href"));
-            manga.title = urlElement.attr("title");
-        }
-        return manga;
-    }
-
-    @Override
-    protected String parseNextPopularMangasUrl(Document parsedHtml, MangasPage page) {
-        Element next = Parser.element(parsedHtml, "div.next-page > a.next");
-        return next != null ? String.format(POPULAR_MANGAS_URL, next.attr("href")) : null;
-    }
-
-    @Override
-    protected List<Manga> parseSearchFromHtml(Document parsedHtml) {
-        List<Manga> mangaList = new ArrayList<>();
-
-        Elements mangaHtmlBlocks = parsedHtml.select("div.result_search > dl");
-        for (Element currentHtmlBlock : mangaHtmlBlocks) {
-            Manga currentManga = constructSearchMangaFromHtmlBlock(currentHtmlBlock);
-            mangaList.add(currentManga);
-        }
-        return mangaList;
-    }
-
-    private Manga constructSearchMangaFromHtmlBlock(Element htmlBlock) {
-        Manga manga = new Manga();
-        manga.source = getId();
-
-        Element urlElement = Parser.element(htmlBlock, "a.manga_info");
-        if (urlElement != null) {
-            manga.setUrl(urlElement.attr("href"));
-            manga.title = urlElement.text();
-        }
-        return manga;
-    }
-
-    @Override
-    protected String parseNextSearchUrl(Document parsedHtml, MangasPage page, String query) {
-        Element next = Parser.element(parsedHtml, "div.next-page > a.next");
-        return next != null ? BASE_URL + next.attr("href") : null;
-    }
-
-    private long parseUpdateFromElement(Element updateElement) {
-        String updatedDateAsString = updateElement.text();
-
-        if (updatedDateAsString.contains("Today")) {
-            Calendar today = Calendar.getInstance();
-            today.set(Calendar.HOUR_OF_DAY, 0);
-            today.set(Calendar.MINUTE, 0);
-            today.set(Calendar.SECOND, 0);
-            today.set(Calendar.MILLISECOND, 0);
-
-            try {
-                Date withoutDay = new SimpleDateFormat("MMM d, yyyy h:mma", Locale.ENGLISH).parse(updatedDateAsString.replace("Today", ""));
-                return today.getTimeInMillis() + withoutDay.getTime();
-            } catch (ParseException e) {
-                return today.getTimeInMillis();
-            }
-        } else if (updatedDateAsString.contains("Yesterday")) {
-            Calendar yesterday = Calendar.getInstance();
-            yesterday.add(Calendar.DATE, -1);
-            yesterday.set(Calendar.HOUR_OF_DAY, 0);
-            yesterday.set(Calendar.MINUTE, 0);
-            yesterday.set(Calendar.SECOND, 0);
-            yesterday.set(Calendar.MILLISECOND, 0);
-
-            try {
-                Date withoutDay = new SimpleDateFormat("MMM d, yyyy h:mma", Locale.ENGLISH).parse(updatedDateAsString.replace("Yesterday", ""));
-                return yesterday.getTimeInMillis() + withoutDay.getTime();
-            } catch (ParseException e) {
-                return yesterday.getTimeInMillis();
-            }
-        } else {
-            try {
-                Date specificDate = new SimpleDateFormat("MMM d, yyyy h:mma", Locale.ENGLISH).parse(updatedDateAsString);
-
-                return specificDate.getTime();
-            } catch (ParseException e) {
-                // Do Nothing.
-            }
-        }
-
-        return 0;
-    }
-
-    public Manga parseHtmlToManga(String mangaUrl, String unparsedHtml) {
-        int beginIndex = unparsedHtml.indexOf("<ul class=\"detail_topText\">");
-        int endIndex = unparsedHtml.indexOf("</ul>", beginIndex);
-        String trimmedHtml = unparsedHtml.substring(beginIndex, endIndex);
-
-        Document parsedDocument = Jsoup.parse(trimmedHtml);
-        Element detailElement = parsedDocument.select("ul.detail_topText").first();
-
-        Manga manga = Manga.create(mangaUrl);
-        manga.author = Parser.text(parsedDocument, "a[href^=http://www.mangahere.co/author/]");
-        manga.artist = Parser.text(parsedDocument, "a[href^=http://www.mangahere.co/artist/]");
-
-        String description = Parser.text(detailElement, "#show");
-        if (description != null) {
-            manga.description = description.substring(0, description.length() - "Show less".length());
-        }
-        String genres = Parser.text(detailElement, "li:eq(3)");
-        if (genres != null) {
-            manga.genre = genres.substring("Genre(s):".length());
-        }
-        manga.status = parseStatus(Parser.text(detailElement, "li:eq(6)"));
-
-        beginIndex = unparsedHtml.indexOf("<img");
-        endIndex = unparsedHtml.indexOf("/>", beginIndex);
-        trimmedHtml = unparsedHtml.substring(beginIndex, endIndex + 2);
-
-        parsedDocument = Jsoup.parse(trimmedHtml);
-        manga.thumbnail_url = Parser.src(parsedDocument, "img");
-
-        manga.initialized = true;
-        return manga;
-    }
-
-    private int parseStatus(String status) {
-        if (status.contains("Ongoing")) {
-            return Manga.ONGOING;
-        }
-        if (status.contains("Completed")) {
-            return Manga.COMPLETED;
-        }
-        return Manga.UNKNOWN;
-    }
-
-    @Override
-    public List<Chapter> parseHtmlToChapters(String unparsedHtml) {
-        int beginIndex = unparsedHtml.indexOf("<ul>");
-        int endIndex = unparsedHtml.indexOf("</ul>", beginIndex);
-        String trimmedHtml = unparsedHtml.substring(beginIndex, endIndex);
-
-        Document parsedDocument = Jsoup.parse(trimmedHtml);
-
-        List<Chapter> chapterList = new ArrayList<>();
-
-        for (Element chapterElement : parsedDocument.getElementsByTag("li")) {
-            Chapter currentChapter = constructChapterFromHtmlBlock(chapterElement);
-            chapterList.add(currentChapter);
-        }
-        return chapterList;
-    }
-
-    private Chapter constructChapterFromHtmlBlock(Element chapterElement) {
-        Chapter chapter = Chapter.create();
-
-        Element urlElement = chapterElement.select("a").first();
-        Element dateElement = chapterElement.select("span.right").first();
-
-        if (urlElement != null) {
-            chapter.setUrl(urlElement.attr("href"));
-            chapter.name = urlElement.text();
-        }
-        if (dateElement != null) {
-            chapter.date_upload = parseDateFromElement(dateElement);
-        }
-        return chapter;
-    }
-
-    private long parseDateFromElement(Element dateElement) {
-        String dateAsString = dateElement.text();
-
-        if (dateAsString.contains("Today")) {
-            Calendar today = Calendar.getInstance();
-            today.set(Calendar.HOUR_OF_DAY, 0);
-            today.set(Calendar.MINUTE, 0);
-            today.set(Calendar.SECOND, 0);
-            today.set(Calendar.MILLISECOND, 0);
-
-            try {
-                Date withoutDay = new SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH).parse(dateAsString.replace("Today", ""));
-                return today.getTimeInMillis() + withoutDay.getTime();
-            } catch (ParseException e) {
-                return today.getTimeInMillis();
-            }
-        } else if (dateAsString.contains("Yesterday")) {
-            Calendar yesterday = Calendar.getInstance();
-            yesterday.add(Calendar.DATE, -1);
-            yesterday.set(Calendar.HOUR_OF_DAY, 0);
-            yesterday.set(Calendar.MINUTE, 0);
-            yesterday.set(Calendar.SECOND, 0);
-            yesterday.set(Calendar.MILLISECOND, 0);
-
-            try {
-                Date withoutDay = new SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH).parse(dateAsString.replace("Yesterday", ""));
-                return yesterday.getTimeInMillis() + withoutDay.getTime();
-            } catch (ParseException e) {
-                return yesterday.getTimeInMillis();
-            }
-        } else {
-            try {
-                Date date = new SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH).parse(dateAsString);
-
-                return date.getTime();
-            } catch (ParseException e) {
-                // Do Nothing.
-            }
-        }
-        return 0;
-    }
-
-    @Override
-    public List<String> parseHtmlToPageUrls(String unparsedHtml) {
-        int beginIndex = unparsedHtml.indexOf("<div class=\"go_page clearfix\">");
-        int endIndex = unparsedHtml.indexOf("</div>", beginIndex);
-        String trimmedHtml = unparsedHtml.substring(beginIndex, endIndex);
-
-        Document parsedDocument = Jsoup.parse(trimmedHtml);
-
-        List<String> pageUrlList = new ArrayList<>();
-
-        Elements pageUrlElements = parsedDocument.select("select.wid60").first().getElementsByTag("option");
-        for (Element pageUrlElement : pageUrlElements) {
-            pageUrlList.add(pageUrlElement.attr("value"));
-        }
-
-        return pageUrlList;
-    }
-
-    @Override
-    public String parseHtmlToImageUrl(String unparsedHtml) {
-        int beginIndex = unparsedHtml.indexOf("<section class=\"read_img\" id=\"viewer\">");
-        int endIndex = unparsedHtml.indexOf("</section>", beginIndex);
-        String trimmedHtml = unparsedHtml.substring(beginIndex, endIndex);
-
-        Document parsedDocument = Jsoup.parse(trimmedHtml);
-
-        Element imageElement = parsedDocument.getElementById("image");
-
-        return imageElement.attr("src");
-    }
-
-}

app/src/main/java/eu/kanade/tachiyomi/data/source/online/english/ReadMangaToday.java

@@ -1,290 +0,0 @@
-package eu.kanade.tachiyomi.data.source.online.english;
-
-import android.content.Context;
-import android.net.Uri;
-
-import com.google.gson.Gson;
-import com.google.gson.JsonArray;
-import com.google.gson.JsonElement;
-import com.google.gson.JsonObject;
-import com.google.gson.JsonParser;
-
-import org.jsoup.Jsoup;
-import org.jsoup.nodes.Document;
-import org.jsoup.nodes.Element;
-import org.jsoup.select.Elements;
-
-import java.util.ArrayList;
-import java.util.Calendar;
-import java.util.List;
-
-import eu.kanade.tachiyomi.data.database.models.Chapter;
-import eu.kanade.tachiyomi.data.database.models.Manga;
-import eu.kanade.tachiyomi.data.source.Language;
-import eu.kanade.tachiyomi.data.source.LanguageKt;
-import eu.kanade.tachiyomi.data.source.base.Source;
-import eu.kanade.tachiyomi.data.source.model.MangasPage;
-import eu.kanade.tachiyomi.util.Parser;
-import okhttp3.Headers;
-import rx.Observable;
-import rx.functions.Action1;
-import rx.functions.Func1;
-
-public class ReadMangaToday extends Source {
-    public static final String NAME = "ReadMangaToday";
-    public static final String BASE_URL = "http://www.readmanga.today";
-    public static final String POPULAR_MANGAS_URL = BASE_URL + "/hot-manga/%s";
-    public static final String SEARCH_URL = BASE_URL + "/service/search?q=%s";
-
-    private static JsonParser parser = new JsonParser();
-    private static Gson gson = new Gson();
-
-    public ReadMangaToday(Context context) {
-        super(context);
-    }
-
-    @Override
-    public String getName() {
-        return NAME;
-    }
-
-    @Override
-    public String getBaseUrl() {
-        return BASE_URL;
-    }
-
-    @Override
-    protected String getInitialPopularMangasUrl() {
-        return String.format(POPULAR_MANGAS_URL, "");
-    }
-
-    @Override
-    protected String getInitialSearchUrl(String query) {
-        return String.format(SEARCH_URL, Uri.encode(query), 1);
-    }
-
-    @Override
-    public Language getLang() {
-        return LanguageKt.getEN();
-    }
-
-    @Override
-    public List<Manga> parsePopularMangasFromHtml(Document parsedHtml) {
-        List<Manga> mangaList = new ArrayList<>();
-
-        for (Element currentHtmlBlock : parsedHtml.select("div.hot-manga > div.style-list > div.box")) {
-            Manga currentManga = constructPopularMangaFromHtmlBlock(currentHtmlBlock);
-            mangaList.add(currentManga);
-        }
-        return mangaList;
-    }
-
-    private Manga constructPopularMangaFromHtmlBlock(Element htmlBlock) {
-        Manga manga = new Manga();
-        manga.source = getId();
-
-        Element urlElement = Parser.element(htmlBlock, "div.title > h2 > a");
-        if (urlElement != null) {
-            manga.setUrl(urlElement.attr("href"));
-            manga.title = urlElement.attr("title");
-        }
-        return manga;
-    }
-
-    @Override
-    protected String parseNextPopularMangasUrl(Document parsedHtml, MangasPage page) {
-        Element next = Parser.element(parsedHtml, "div.hot-manga > ul.pagination > li > a:contains(»)");
-        return next != null ? next.attr("href") : null;
-    }
-
-    @Override
-    public Observable<MangasPage> searchMangasFromNetwork(final MangasPage page, String query) {
-        return networkService
-                .requestBody(searchMangaRequest(page, query), networkService.getDefaultClient())
-                .doOnNext(new Action1<String>() {
-                    @Override
-                    public void call(String doc) {
-                        page.mangas = ReadMangaToday.this.parseSearchFromJson(doc);
-                    }
-                })
-                .map(new Func1<String, MangasPage>() {
-                    @Override
-                    public MangasPage call(String response) {
-                        return page;
-                    }
-                });
-    }
-
-    @Override
-    protected Headers.Builder headersBuilder() {
-        return super.headersBuilder().add("X-Requested-With", "XMLHttpRequest");
-    }
-
-    protected List<Manga> parseSearchFromJson(String unparsedJson) {
-        List<Manga> mangaList = new ArrayList<>();
-
-        JsonArray mangasArray = parser.parse(unparsedJson).getAsJsonArray();
-
-        for (JsonElement mangaElement : mangasArray) {
-            Manga currentManga = constructSearchMangaFromJsonObject(mangaElement.getAsJsonObject());
-            mangaList.add(currentManga);
-        }
-        return mangaList;
-    }
-
-    private Manga constructSearchMangaFromJsonObject(JsonObject jsonObject) {
-        Manga manga = new Manga();
-        manga.source = getId();
-
-        manga.setUrl(gson.fromJson(jsonObject.get("url"), String.class));
-        manga.title = gson.fromJson(jsonObject.get("title"), String.class);
-
-        return manga;
-    }
-
-    @Override
-    protected List<Manga> parseSearchFromHtml(Document parsedHtml) {
-        return null;
-    }
-
-    @Override
-    protected String parseNextSearchUrl(Document parsedHtml, MangasPage page, String query) {
-        return null;
-    }
-
-    public Manga parseHtmlToManga(String mangaUrl, String unparsedHtml) {
-        int beginIndex = unparsedHtml.indexOf("<!-- content start -->");
-        int endIndex = unparsedHtml.indexOf("<!-- /content-end -->", beginIndex);
-        String trimmedHtml = unparsedHtml.substring(beginIndex, endIndex);
-
-        Document parsedDocument = Jsoup.parse(trimmedHtml);
-        Element detailElement = parsedDocument.select("div.movie-meta").first();
-
-        Manga manga = Manga.create(mangaUrl);
-        for (Element castHtmlBlock : parsedDocument.select("div.cast ul.cast-list > li")) {
-            String name = Parser.text(castHtmlBlock, "ul > li > a");
-            String role = Parser.text(castHtmlBlock, "ul > li:eq(1)");
-            if (role.equals("Author")) {
-                manga.author = name;
-            } else if (role.equals("Artist")) {
-                manga.artist = name;
-            }
-        }
-
-        String description = Parser.text(detailElement, "li.movie-detail");
-        if (description != null) {
-            manga.description = description;
-        }
-        String genres = Parser.text(detailElement, "dl.dl-horizontal > dd:eq(5)");
-        if (genres != null) {
-            manga.genre = genres;
-        }
-        manga.status = parseStatus(Parser.text(detailElement, "dl.dl-horizontal > dd:eq(3)"));
-        manga.thumbnail_url = Parser.src(detailElement, "img.img-responsive");
-
-        manga.initialized = true;
-        return manga;
-    }
-
-    private int parseStatus(String status) {
-        if (status.contains("Ongoing")) {
-            return Manga.ONGOING;
-        } else if (status.contains("Completed")) {
-            return Manga.COMPLETED;
-        }
-        return Manga.UNKNOWN;
-    }
-
-    @Override
-    public List<Chapter> parseHtmlToChapters(String unparsedHtml) {
-        int beginIndex = unparsedHtml.indexOf("<!-- content start -->");
-        int endIndex = unparsedHtml.indexOf("<!-- /content-end -->", beginIndex);
-        String trimmedHtml = unparsedHtml.substring(beginIndex, endIndex);
-
-        Document parsedDocument = Jsoup.parse(trimmedHtml);
-
-        List<Chapter> chapterList = new ArrayList<>();
-
-        for (Element chapterElement : parsedDocument.select("ul.chp_lst > li")) {
-            Chapter currentChapter = constructChapterFromHtmlBlock(chapterElement);
-            chapterList.add(currentChapter);
-        }
-        return chapterList;
-    }
-
-    private Chapter constructChapterFromHtmlBlock(Element chapterElement) {
-        Chapter chapter = Chapter.create();
-
-        Element urlElement = chapterElement.select("a").first();
-        Element dateElement = chapterElement.select("span.dte").first();
-
-        if (urlElement != null) {
-            chapter.setUrl(urlElement.attr("href"));
-            chapter.name = urlElement.select("span.val").text();
-        }
-        if (dateElement != null) {
-            chapter.date_upload = parseDateFromElement(dateElement);
-        }
-        return chapter;
-    }
-
-    private long parseDateFromElement(Element dateElement) {
-        String dateAsString = dateElement.text();
-        String[] dateWords = dateAsString.split(" ");
-
-        if (dateWords.length == 3) {
-            int timeAgo = Integer.parseInt(dateWords[0]);
-            Calendar date = Calendar.getInstance();
-
-            if (dateWords[1].contains("Minute")) {
-                date.add(Calendar.MINUTE, - timeAgo);
-            } else if (dateWords[1].contains("Hour")) {
-                date.add(Calendar.HOUR_OF_DAY, - timeAgo);
-            } else if (dateWords[1].contains("Day")) {
-                date.add(Calendar.DAY_OF_YEAR, -timeAgo);
-            } else if (dateWords[1].contains("Week")) {
-                date.add(Calendar.WEEK_OF_YEAR, -timeAgo);
-            } else if (dateWords[1].contains("Month")) {
-                date.add(Calendar.MONTH, -timeAgo);
-            } else if (dateWords[1].contains("Year")) {
-                date.add(Calendar.YEAR, -timeAgo);
-            }
-
-            return date.getTimeInMillis();
-        }
-
-        return 0;
-    }
-
-    @Override
-    public List<String> parseHtmlToPageUrls(String unparsedHtml) {
-        int beginIndex = unparsedHtml.indexOf("<!-- content start -->");
-        int endIndex = unparsedHtml.indexOf("<!-- /content-end -->", beginIndex);
-        String trimmedHtml = unparsedHtml.substring(beginIndex, endIndex);
-
-        Document parsedDocument = Jsoup.parse(trimmedHtml);
-
-        List<String> pageUrlList = new ArrayList<>();
-
-        Elements pageUrlElements = parsedDocument.select("ul.list-switcher-2 > li > select.jump-menu").first().getElementsByTag("option");
-        for (Element pageUrlElement : pageUrlElements) {
-            pageUrlList.add(pageUrlElement.attr("value"));
-        }
-
-        return pageUrlList;
-    }
-
-    @Override
-    public String parseHtmlToImageUrl(String unparsedHtml) {
-        int beginIndex = unparsedHtml.indexOf("<!-- content start -->");
-        int endIndex = unparsedHtml.indexOf("<!-- /content-end -->", beginIndex);
-        String trimmedHtml = unparsedHtml.substring(beginIndex, endIndex);
-
-        Document parsedDocument = Jsoup.parse(trimmedHtml);
-
-        Element imageElement = Parser.element(parsedDocument, "img.img-responsive-2");
-
-        return imageElement.attr("src");
-    }
-
-}

app/src/main/java/eu/kanade/tachiyomi/data/source/online/russian/Mangachan.java

@@ -1,240 +0,0 @@
-package eu.kanade.tachiyomi.data.source.online.russian;
-
-import android.content.Context;
-import android.net.Uri;
-
-import org.jsoup.Jsoup;
-import org.jsoup.nodes.Document;
-import org.jsoup.nodes.Element;
-
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Locale;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import eu.kanade.tachiyomi.data.database.models.Chapter;
-import eu.kanade.tachiyomi.data.database.models.Manga;
-import eu.kanade.tachiyomi.data.source.Language;
-import eu.kanade.tachiyomi.data.source.LanguageKt;
-import eu.kanade.tachiyomi.data.source.base.Source;
-import eu.kanade.tachiyomi.data.source.model.MangasPage;
-import eu.kanade.tachiyomi.data.source.model.Page;
-import eu.kanade.tachiyomi.util.Parser;
-
-public class Mangachan extends Source {
-
-    public static final String NAME = "Mangachan";
-    public static final String BASE_URL = "http://mangachan.ru";
-    public static final String POPULAR_MANGAS_URL = BASE_URL + "/mostfavorites";
-    public static final String SEARCH_URL = BASE_URL + "/?do=search&subaction=search&story=%s";
-
-    public Mangachan(Context context) {
-        super(context);
-    }
-
-    @Override
-    public Language getLang() {
-        return LanguageKt.getRU();
-    }
-
-    @Override
-    public String getName() {
-        return NAME;
-    }
-
-    @Override
-    public String getBaseUrl() {
-        return BASE_URL;
-    }
-
-    @Override
-    protected String getInitialPopularMangasUrl() {
-        return POPULAR_MANGAS_URL;
-    }
-
-    @Override
-    protected String getInitialSearchUrl(String query) {
-        return String.format(SEARCH_URL, Uri.encode(query));
-    }
-
-    @Override
-    protected List<Manga> parsePopularMangasFromHtml(Document parsedHtml) {
-        List<Manga> mangaList = new ArrayList<>();
-
-        for (Element currentHtmlBlock : parsedHtml.select("div.content_row")) {
-            Manga manga = constructPopularMangaFromHtml(currentHtmlBlock);
-            mangaList.add(manga);
-        }
-
-        return mangaList;
-    }
-
-    private Manga constructPopularMangaFromHtml(Element currentHtmlBlock) {
-        Manga manga = new Manga();
-        manga.source = getId();
-
-        Element urlElement = currentHtmlBlock.getElementsByTag("h2").select("a").first();
-        Element imgElement = currentHtmlBlock.getElementsByClass("manga_images").select("img").first();
-
-        if (urlElement != null) {
-            manga.setUrl(urlElement.attr("href"));
-            manga.title = urlElement.text();
-        }
-
-        if (imgElement != null) {
-            manga.thumbnail_url = BASE_URL + imgElement.attr("src");
-        }
-
-        return manga;
-    }
-
-    @Override
-    protected String parseNextPopularMangasUrl(Document parsedHtml, MangasPage page) {
-        String path = Parser.href(parsedHtml, "a:contains(Вперед)");
-        return path != null ? POPULAR_MANGAS_URL + path : null;
-    }
-
-    @Override
-    protected List<Manga> parseSearchFromHtml(Document parsedHtml) {
-        return parsePopularMangasFromHtml(parsedHtml);
-    }
-
-    @Override
-    protected String parseNextSearchUrl(Document parsedHtml, MangasPage page, String query) {
-        return null;
-    }
-
-    @Override
-    protected Manga parseHtmlToManga(String mangaUrl, String unparsedHtml) {
-        Document parsedDocument = Jsoup.parse(unparsedHtml);
-
-        Element infoElement = parsedDocument.getElementsByClass("mangatitle").first();
-        String description = parsedDocument.getElementById("description").text();
-
-        Manga manga = Manga.create(mangaUrl);
-
-        manga.author = infoElement.select("tr:eq(2) td:eq(1)").text();
-        manga.genre = infoElement.select("tr:eq(5) td:eq(1)").text();
-        manga.status = parseStatus(infoElement.select("tr:eq(4) td:eq(1)").text());
-
-        manga.description = description.replaceAll("Прислать описание", "");
-
-        manga.initialized = true;
-        return manga;
-    }
-
-    private int parseStatus(String status) {
-        if (status.contains("перевод продолжается")) {
-            return Manga.ONGOING;
-        } else if (status.contains("перевод завершен")) {
-            return Manga.COMPLETED;
-        } else return Manga.UNKNOWN;
-    }
-
-    @Override
-    protected List<Chapter> parseHtmlToChapters(String unparsedHtml) {
-        Document parsedDocument = Jsoup.parse(unparsedHtml);
-        List<Chapter> chapterList = new ArrayList<>();
-
-        for (Element chapterElement : parsedDocument.select("table.table_cha tr:gt(1)")) {
-            Chapter chapter = constructChapterFromHtmlBlock(chapterElement);
-            chapterList.add(chapter);
-        }
-        return chapterList;
-    }
-
-    private Chapter constructChapterFromHtmlBlock(Element chapterElement) {
-        Chapter chapter = Chapter.create();
-
-        Element urlElement = chapterElement.select("a").first();
-        String date = Parser.text(chapterElement, "div.date");
-
-        if (urlElement != null) {
-            chapter.name = urlElement.text();
-            chapter.url = urlElement.attr("href");
-        }
-
-        if (date != null) {
-            try {
-                chapter.date_upload = new SimpleDateFormat("yyyy-MM-dd", Locale.ENGLISH).parse(date).getTime();
-            } catch (ParseException e) { /* Ignore */ }
-        }
-        return chapter;
-    }
-
-    // Without this extra chapters are in the wrong place in the list
-    @Override
-    public void parseChapterNumber(Chapter chapter) {
-        // For chapters with url like /online/254903-fairy-tail_v56_ch474.html
-        String url = chapter.url.replace(".html", "");
-        Pattern pattern = Pattern.compile("\\d+_ch[\\d.]+");
-        Matcher matcher = pattern.matcher(url);
-
-        if (matcher.find()) {
-            String[] parts = matcher.group().split("_ch");
-            chapter.chapter_number = Float.parseFloat(parts[0] + "." + AddZero(parts[1]));
-        } else { // For chapters with url like /online/61216-3298.html
-            String name = chapter.name;
-            name = name.replaceAll("[\\s\\d\\w\\W]+v", "");
-            String volume = name.substring(0, name.indexOf(" - "));
-            String[] parts = name.replaceFirst("\\d+ - ", "").split(" ");
-
-            chapter.chapter_number = Float.parseFloat(volume + "." + AddZero(parts[0]));
-        }
-    }
-
-    private String AddZero(String num) {
-        if (Float.parseFloat(num) < 1000f) {
-            num = "0" + num.replace(".", "");
-        }
-        if (Float.parseFloat(num) < 100f) {
-            num = "0" + num.replace(".", "");
-        }
-        if (Float.parseFloat(num) < 10f) {
-            num = "0" + num.replace(".", "");
-        }
-        return num;
-    }
-
-    @Override
-    protected List<String> parseHtmlToPageUrls(String unparsedHtml) {
-        ArrayList<String> pages = new ArrayList<>();
-
-        int beginIndex = unparsedHtml.indexOf("fullimg\":[");
-        int endIndex = unparsedHtml.indexOf(']', beginIndex);
-
-        String trimmedHtml = unparsedHtml.substring(beginIndex + 10, endIndex);
-        trimmedHtml = trimmedHtml.replaceAll("\"", "");
-
-        String[] pageUrls = trimmedHtml.split(",");
-        for (int i = 0; i < pageUrls.length; i++) {
-            pages.add("");
-        }
-
-        return pages;
-    }
-
-    @Override
-    protected List<Page> parseFirstPage(List<? extends Page> pages, String unparsedHtml) {
-        int beginIndex = unparsedHtml.indexOf("fullimg\":[");
-        int endIndex = unparsedHtml.indexOf(']', beginIndex);
-
-        String trimmedHtml = unparsedHtml.substring(beginIndex + 10, endIndex);
-        trimmedHtml = trimmedHtml.replaceAll("\"", "");
-
-        String[] pageUrls = trimmedHtml.split(",");
-        for (int i = 0; i < pageUrls.length; i++) {
-            pages.get(i).setImageUrl(pageUrls[i].replaceAll("im.?\\.", ""));
-        }
-
-        return (List<Page>) pages;
-    }
-
-    @Override
-    protected String parseHtmlToImageUrl(String unparsedHtml) {
-        return null;
-    }
-}

app/src/main/java/eu/kanade/tachiyomi/data/source/online/russian/Mintmanga.java

@@ -1,225 +0,0 @@
-package eu.kanade.tachiyomi.data.source.online.russian;
-
-import android.content.Context;
-import android.net.Uri;
-
-import org.jsoup.Jsoup;
-import org.jsoup.nodes.Document;
-import org.jsoup.nodes.Element;
-
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Locale;
-
-import eu.kanade.tachiyomi.data.database.models.Chapter;
-import eu.kanade.tachiyomi.data.database.models.Manga;
-import eu.kanade.tachiyomi.data.source.Language;
-import eu.kanade.tachiyomi.data.source.LanguageKt;
-import eu.kanade.tachiyomi.data.source.base.Source;
-import eu.kanade.tachiyomi.data.source.model.MangasPage;
-import eu.kanade.tachiyomi.data.source.model.Page;
-import eu.kanade.tachiyomi.util.Parser;
-
-public class Mintmanga extends Source {
-
-    public static final String NAME = "Mintmanga";
-    public static final String BASE_URL = "http://mintmanga.com";
-    public static final String POPULAR_MANGAS_URL = BASE_URL + "/list?sortType=rate";
-    public static final String SEARCH_URL = BASE_URL + "/search?q=%s";
-
-    public Mintmanga(Context context) {
-        super(context);
-    }
-
-    @Override
-    public Language getLang() {
-        return LanguageKt.getRU();
-    }
-
-    @Override
-    public String getName() {
-        return NAME;
-    }
-
-    @Override
-    public String getBaseUrl() {
-        return BASE_URL;
-    }
-
-    @Override
-    protected String getInitialPopularMangasUrl() {
-        return POPULAR_MANGAS_URL;
-    }
-
-    @Override
-    protected String getInitialSearchUrl(String query) {
-        return String.format(SEARCH_URL, Uri.encode(query));
-    }
-
-    @Override
-    protected List<Manga> parsePopularMangasFromHtml(Document parsedHtml) {
-        List<Manga> mangaList = new ArrayList<>();
-
-        for (Element currentHtmlBlock : parsedHtml.select("div.desc")) {
-            Manga manga = constructPopularMangaFromHtml(currentHtmlBlock);
-            mangaList.add(manga);
-        }
-
-        return mangaList;
-    }
-
-    private Manga constructPopularMangaFromHtml(Element currentHtmlBlock) {
-        Manga manga = new Manga();
-        manga.source = getId();
-
-        Element urlElement = currentHtmlBlock.getElementsByTag("h3").select("a").first();
-
-        if (urlElement != null) {
-            manga.setUrl(urlElement.attr("href"));
-            manga.title = urlElement.text();
-        }
-
-        return manga;
-    }
-
-    @Override
-    protected String parseNextPopularMangasUrl(Document parsedHtml, MangasPage page) {
-        String path = Parser.href(parsedHtml, "a:contains(→)");
-        return path != null ? BASE_URL + path : null;
-    }
-
-    @Override
-    protected List<Manga> parseSearchFromHtml(Document parsedHtml) {
-        return parsePopularMangasFromHtml(parsedHtml);
-    }
-
-    @Override
-    protected String parseNextSearchUrl(Document parsedHtml, MangasPage page, String query) {
-        return null;
-    }
-
-    @Override
-    protected Manga parseHtmlToManga(String mangaUrl, String unparsedHtml) {
-        Document parsedDocument = Jsoup.parse(unparsedHtml);
-        Element infoElement = parsedDocument.select("div.leftContent").first();
-
-        Manga manga = Manga.create(mangaUrl);
-        manga.title = Parser.text(infoElement, "span.eng-name");
-        manga.author = Parser.text(infoElement, "span.elem_author ");
-        manga.genre = Parser.allText(infoElement, "span.elem_genre ").replaceAll(" ,", ",");
-        manga.description = Parser.allText(infoElement, "div.manga-description");
-        if (Parser.text(infoElement, "h1.names:contains(Сингл)") != null) {
-            manga.status = Manga.COMPLETED;
-        } else {
-            manga.status = parseStatus(Parser.text(infoElement, "p:has(b:contains(Перевод:))"));
-        }
-
-        String thumbnail = Parser.element(infoElement, "img").attr("data-full");
-        if (thumbnail != null) {
-            manga.thumbnail_url = thumbnail;
-        }
-
-        manga.initialized = true;
-        return manga;
-    }
-
-    private int parseStatus(String status) {
-        if (status.contains("продолжается")) {
-            return Manga.ONGOING;
-        }
-        if (status.contains("завершен")) {
-            return Manga.COMPLETED;
-        }
-        return Manga.UNKNOWN;
-    }
-
-    @Override
-    protected List<Chapter> parseHtmlToChapters(String unparsedHtml) {
-        Document parsedDocument = Jsoup.parse(unparsedHtml);
-        List<Chapter> chapterList = new ArrayList<>();
-
-        for (Element chapterElement : parsedDocument.select("div.chapters-link tbody tr")) {
-            Chapter chapter = constructChapterFromHtmlBlock(chapterElement);
-            chapterList.add(chapter);
-        }
-
-        return chapterList;
-    }
-
-    private Chapter constructChapterFromHtmlBlock(Element chapterElement) {
-        Chapter chapter = Chapter.create();
-
-        Element urlElement = Parser.element(chapterElement, "a");
-        String date = Parser.text(chapterElement, "td:eq(1)");
-
-        if (urlElement != null) {
-            chapter.setUrl(urlElement.attr("href") + "?mature=1");
-            chapter.name = urlElement.text().replaceAll(" новое", "");
-        }
-
-        if (date != null) {
-            try {
-                chapter.date_upload = new SimpleDateFormat("dd/MM/yy", Locale.ENGLISH).parse(date).getTime();
-            } catch (ParseException e) { /* Ignore */ }
-        }
-        return chapter;
-    }
-
-    // Without this extra chapters are in the wrong place in the list
-    @Override
-    public void parseChapterNumber(Chapter chapter) {
-        String url = chapter.url.replace("?mature=1", "");
-
-        String[] urlParts = url.replaceAll("/[\\w\\d]+/vol", "").split("/");
-        if (Float.parseFloat(urlParts[1]) < 1000f) {
-            urlParts[1] = "0" + urlParts[1];
-        }
-        if (Float.parseFloat(urlParts[1]) < 100f) {
-            urlParts[1] = "0" + urlParts[1];
-        }
-        if (Float.parseFloat(urlParts[1]) < 10f) {
-            urlParts[1] = "0" + urlParts[1];
-        }
-
-        chapter.chapter_number = Float.parseFloat(urlParts[0] + "." + urlParts[1]);
-    }
-
-    @Override
-    protected List<String> parseHtmlToPageUrls(String unparsedHtml) {
-        ArrayList<String> pages = new ArrayList<>();
-
-        int beginIndex = unparsedHtml.indexOf("rm_h.init( [");
-        int endIndex = unparsedHtml.indexOf("], 0, false);", beginIndex);
-
-        String trimmedHtml = unparsedHtml.substring(beginIndex + 13, endIndex);
-        trimmedHtml = trimmedHtml.replaceAll("[\"']", "");
-        String[] pageUrls = trimmedHtml.split("],\\[");
-        for (int i = 0; i < pageUrls.length; i++) {
-            pages.add("");
-        }
-        return pages;
-    }
-
-    @Override
-    protected List<Page> parseFirstPage(List<? extends Page> pages, String unparsedHtml) {
-        int beginIndex = unparsedHtml.indexOf("rm_h.init( [");
-        int endIndex = unparsedHtml.indexOf("], 0, false);", beginIndex);
-
-        String trimmedHtml = unparsedHtml.substring(beginIndex + 13, endIndex);
-        trimmedHtml = trimmedHtml.replaceAll("[\"']", "");
-        String[] pageUrls = trimmedHtml.split("],\\[");
-        for (int i = 0; i < pageUrls.length; i++) {
-            String[] urlParts = pageUrls[i].split(","); // auto/06/35,http://e4.adultmanga.me/,/55/01.png
-            String page = urlParts[1] + urlParts[0] + urlParts[2];
-            pages.get(i).setImageUrl(page);
-        }
-        return (List<Page>) pages;
-    }
-
-    @Override
-    protected String parseHtmlToImageUrl(String unparsedHtml) {
-        return null;
-    }
-}

app/src/main/java/eu/kanade/tachiyomi/data/source/online/russian/Readmanga.java

@@ -1,225 +0,0 @@
-package eu.kanade.tachiyomi.data.source.online.russian;
-
-import android.content.Context;
-import android.net.Uri;
-
-import org.jsoup.Jsoup;
-import org.jsoup.nodes.Document;
-import org.jsoup.nodes.Element;
-
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Locale;
-
-import eu.kanade.tachiyomi.data.database.models.Chapter;
-import eu.kanade.tachiyomi.data.database.models.Manga;
-import eu.kanade.tachiyomi.data.source.Language;
-import eu.kanade.tachiyomi.data.source.LanguageKt;
-import eu.kanade.tachiyomi.data.source.base.Source;
-import eu.kanade.tachiyomi.data.source.model.MangasPage;
-import eu.kanade.tachiyomi.data.source.model.Page;
-import eu.kanade.tachiyomi.util.Parser;
-
-public class Readmanga extends Source {
-
-    public static final String NAME = "Readmanga";
-    public static final String BASE_URL = "http://readmanga.me";
-    public static final String POPULAR_MANGAS_URL = BASE_URL + "/list?sortType=rate";
-    public static final String SEARCH_URL = BASE_URL + "/search?q=%s";
-
-    public Readmanga(Context context) {
-        super(context);
-    }
-
-    @Override
-    public Language getLang() {
-        return LanguageKt.getRU();
-    }
-
-    @Override
-    public String getName() {
-        return NAME;
-    }
-
-    @Override
-    public String getBaseUrl() {
-        return BASE_URL;
-    }
-
-    @Override
-    protected String getInitialPopularMangasUrl() {
-        return POPULAR_MANGAS_URL;
-    }
-
-    @Override
-    protected String getInitialSearchUrl(String query) {
-        return String.format(SEARCH_URL, Uri.encode(query));
-    }
-
-    @Override
-    protected List<Manga> parsePopularMangasFromHtml(Document parsedHtml) {
-        List<Manga> mangaList = new ArrayList<>();
-
-        for (Element currentHtmlBlock : parsedHtml.select("div.desc")) {
-            Manga manga = constructPopularMangaFromHtml(currentHtmlBlock);
-            mangaList.add(manga);
-        }
-
-        return mangaList;
-    }
-
-    private Manga constructPopularMangaFromHtml(Element currentHtmlBlock) {
-        Manga manga = new Manga();
-        manga.source = getId();
-
-        Element urlElement = currentHtmlBlock.getElementsByTag("h3").select("a").first();
-
-        if (urlElement != null) {
-            manga.setUrl(urlElement.attr("href"));
-            manga.title = urlElement.text();
-        }
-
-        return manga;
-    }
-
-    @Override
-    protected String parseNextPopularMangasUrl(Document parsedHtml, MangasPage page) {
-        String path = Parser.href(parsedHtml, "a:contains(→)");
-        return path != null ? BASE_URL + path : null;
-    }
-
-    @Override
-    protected List<Manga> parseSearchFromHtml(Document parsedHtml) {
-        return parsePopularMangasFromHtml(parsedHtml);
-    }
-
-    @Override
-    protected String parseNextSearchUrl(Document parsedHtml, MangasPage page, String query) {
-        return null;
-    }
-
-    @Override
-    protected Manga parseHtmlToManga(String mangaUrl, String unparsedHtml) {
-        Document parsedDocument = Jsoup.parse(unparsedHtml);
-        Element infoElement = parsedDocument.select("div.leftContent").first();
-
-        Manga manga = Manga.create(mangaUrl);
-        manga.title = Parser.text(infoElement, "span.eng-name");
-        manga.author = Parser.text(infoElement, "span.elem_author ");
-        manga.genre = Parser.allText(infoElement, "span.elem_genre ").replaceAll(" ,", ",");
-        manga.description = Parser.allText(infoElement, "div.manga-description");
-        if (Parser.text(infoElement, "h1.names:contains(Сингл)") != null) {
-            manga.status = Manga.COMPLETED;
-        } else {
-            manga.status = parseStatus(Parser.text(infoElement, "p:has(b:contains(Перевод:))"));
-        }
-
-        String thumbnail = Parser.element(infoElement, "img").attr("data-full");
-        if (thumbnail != null) {
-            manga.thumbnail_url = thumbnail;
-        }
-
-        manga.initialized = true;
-        return manga;
-    }
-
-    private int parseStatus(String status) {
-        if (status.contains("продолжается")) {
-            return Manga.ONGOING;
-        }
-        if (status.contains("завершен")) {
-            return Manga.COMPLETED;
-        }
-        return Manga.UNKNOWN;
-    }
-
-    @Override
-    protected List<Chapter> parseHtmlToChapters(String unparsedHtml) {
-        Document parsedDocument = Jsoup.parse(unparsedHtml);
-        List<Chapter> chapterList = new ArrayList<>();
-
-        for (Element chapterElement : parsedDocument.select("div.chapters-link tbody tr")) {
-            Chapter chapter = constructChapterFromHtmlBlock(chapterElement);
-            chapterList.add(chapter);
-        }
-
-        return chapterList;
-    }
-
-    private Chapter constructChapterFromHtmlBlock(Element chapterElement) {
-        Chapter chapter = Chapter.create();
-
-        Element urlElement = Parser.element(chapterElement, "a");
-        String date = Parser.text(chapterElement, "td:eq(1)");
-
-        if (urlElement != null) {
-            chapter.setUrl(urlElement.attr("href") + "?mature=1");
-            chapter.name = urlElement.text().replaceAll(" новое", "");
-        }
-
-        if (date != null) {
-            try {
-                chapter.date_upload = new SimpleDateFormat("dd/MM/yy", Locale.ENGLISH).parse(date).getTime();
-            } catch (ParseException e) { /* Ignore */ }
-        }
-        return chapter;
-    }
-
-    // Without this extra chapters are in the wrong place in the list
-    @Override
-    public void parseChapterNumber(Chapter chapter) {
-        String url = chapter.url.replace("?mature=1", "");
-
-        String[] urlParts = url.replaceAll("/[\\w\\d]+/vol", "").split("/");
-        if (Float.parseFloat(urlParts[1]) < 1000f) {
-            urlParts[1] = "0" + urlParts[1];
-        }
-        if (Float.parseFloat(urlParts[1]) < 100f) {
-            urlParts[1] = "0" + urlParts[1];
-        }
-        if (Float.parseFloat(urlParts[1]) < 10f) {
-            urlParts[1] = "0" + urlParts[1];
-        }
-
-        chapter.chapter_number = Float.parseFloat(urlParts[0] + "." + urlParts[1]);
-    }
-
-    @Override
-    protected List<String> parseHtmlToPageUrls(String unparsedHtml) {
-        ArrayList<String> pages = new ArrayList<>();
-
-        int beginIndex = unparsedHtml.indexOf("rm_h.init( [");
-        int endIndex = unparsedHtml.indexOf("], 0, false);", beginIndex);
-
-        String trimmedHtml = unparsedHtml.substring(beginIndex + 13, endIndex);
-        trimmedHtml = trimmedHtml.replaceAll("[\"']", "");
-        String[] pageUrls = trimmedHtml.split("],\\[");
-        for (int i = 0; i < pageUrls.length; i++) {
-            pages.add("");
-        }
-        return pages;
-    }
-
-    @Override
-    protected List<Page> parseFirstPage(List<? extends Page> pages, String unparsedHtml) {
-        int beginIndex = unparsedHtml.indexOf("rm_h.init( [");
-        int endIndex = unparsedHtml.indexOf("], 0, false);", beginIndex);
-
-        String trimmedHtml = unparsedHtml.substring(beginIndex + 13, endIndex);
-        trimmedHtml = trimmedHtml.replaceAll("[\"']", "");
-        String[] pageUrls = trimmedHtml.split("],\\[");
-        for (int i = 0; i < pageUrls.length; i++) {
-            String[] urlParts = pageUrls[i].split(","); // auto/12/56,http://e7.postfact.ru/,/51/01.jpg_res.jpg
-            String page = urlParts[1] + urlParts[0] + urlParts[2];
-            pages.get(i).setImageUrl(page);
-        }
-        return (List<Page>) pages;
-    }
-
-    @Override
-    protected String parseHtmlToImageUrl(String unparsedHtml) {
-        return null;
-    }
-}

app/src/main/java/eu/kanade/tachiyomi/injection/component/AppComponent.kt

@@ -9,6 +9,7 @@ import eu.kanade.tachiyomi.data.library.LibraryUpdateService
 import eu.kanade.tachiyomi.data.mangasync.UpdateMangaSyncService
 import eu.kanade.tachiyomi.data.mangasync.base.MangaSyncService
 import eu.kanade.tachiyomi.data.source.base.Source
+import eu.kanade.tachiyomi.data.source.base.OnlineSource
 import eu.kanade.tachiyomi.data.updater.UpdateDownloader
 import eu.kanade.tachiyomi.injection.module.AppModule
 import eu.kanade.tachiyomi.injection.module.DataModule
@@ -49,6 +50,8 @@ interface AppComponent {
     fun inject(source: Source)
     fun inject(mangaSyncService: MangaSyncService)
 
+    fun inject(onlineSource: OnlineSource)
+
     fun inject(libraryUpdateService: LibraryUpdateService)
     fun inject(downloadService: DownloadService)
     fun inject(updateMangaSyncService: UpdateMangaSyncService)

app/src/main/java/eu/kanade/tachiyomi/ui/catalogue/CataloguePresenter.kt

@@ -8,6 +8,7 @@ import eu.kanade.tachiyomi.data.preference.PreferencesHelper
 import eu.kanade.tachiyomi.data.preference.getOrDefault
 import eu.kanade.tachiyomi.data.source.EN
 import eu.kanade.tachiyomi.data.source.SourceManager
+import eu.kanade.tachiyomi.data.source.base.OnlineSource
 import eu.kanade.tachiyomi.data.source.base.Source
 import eu.kanade.tachiyomi.data.source.model.MangasPage
 import eu.kanade.tachiyomi.ui.base.presenter.BasePresenter
@@ -52,7 +53,7 @@ class CataloguePresenter : BasePresenter<CatalogueFragment>() {
     /**
      * Active source.
      */
-    lateinit var source: Source
+    lateinit var source: OnlineSource
         private set
 
     /**
@@ -163,7 +164,7 @@ class CataloguePresenter : BasePresenter<CatalogueFragment>() {
      *
      * @param source the new active source.
      */
-    fun setActiveSource(source: Source) {
+    fun setActiveSource(source: OnlineSource) {
         prefs.lastUsedCatalogueSource().set(source.id)
         this.source = source
         restartPager()
@@ -222,9 +223,9 @@ class CataloguePresenter : BasePresenter<CatalogueFragment>() {
         }
 
         val observable = if (query.isEmpty())
-            source.pullPopularMangasFromNetwork(nextMangasPage)
+            source.fetchPopularManga(nextMangasPage)
         else
-            source.searchMangasFromNetwork(nextMangasPage, query)
+            source.fetchSearchManga(nextMangasPage, query)
 
         return observable.subscribeOn(Schedulers.io())
                 .doOnNext { lastMangasPage = it }
@@ -268,7 +269,7 @@ class CataloguePresenter : BasePresenter<CatalogueFragment>() {
      * @return an observable of the manga to initialize
      */
     private fun getMangaDetailsObservable(manga: Manga): Observable<Manga> {
-        return source.pullMangaFromNetwork(manga.url)
+        return source.fetchMangaDetails(manga)
                 .flatMap { networkManga ->
                     manga.copyFrom(networkManga)
                     db.insertManga(manga).executeAsBlocking()
@@ -282,13 +283,13 @@ class CataloguePresenter : BasePresenter<CatalogueFragment>() {
      *
      * @return a source.
      */
-    fun getLastUsedSource(): Source {
+    fun getLastUsedSource(): OnlineSource {
         val id = prefs.lastUsedCatalogueSource().get() ?: -1
         val source = sourceManager.get(id)
         if (!isValidSource(source)) {
             return findFirstValidSource()
         }
-        return source!!
+        return source as OnlineSource
     }
 
     /**
@@ -298,10 +299,10 @@ class CataloguePresenter : BasePresenter<CatalogueFragment>() {
      * @return true if the source is valid, false otherwise.
      */
     fun isValidSource(source: Source?): Boolean {
-        if (source == null) return false
+        if (source == null || source !is OnlineSource) return false
 
         return with(source) {
-            if (!isLoginRequired || isLogged) {
+            if (!isLoginRequired() || isLogged()) {
                 true
             } else {
                 prefs.sourceUsername(this) != "" && prefs.sourcePassword(this) != ""
@@ -314,14 +315,14 @@ class CataloguePresenter : BasePresenter<CatalogueFragment>() {
      *
      * @return the index of the first valid source.
      */
-    fun findFirstValidSource(): Source {
-        return sources.find { isValidSource(it) }!!
+    fun findFirstValidSource(): OnlineSource {
+        return sources.first { isValidSource(it) }
     }
 
     /**
      * Returns a list of enabled sources ordered by language and name.
      */
-    private fun getEnabledSources(): List<Source> {
+    private fun getEnabledSources(): List<OnlineSource> {
         val languages = prefs.enabledLanguages().getOrDefault()
 
         // Ensure at least one language
@@ -329,7 +330,7 @@ class CataloguePresenter : BasePresenter<CatalogueFragment>() {
             languages.add(EN.code)
         }
 
-        return sourceManager.getSources()
+        return sourceManager.getOnlineSources()
                 .filter { it.lang.code in languages }
                 .sortedBy { "(${it.lang.code}) ${it.name}" }
     }

app/src/main/java/eu/kanade/tachiyomi/ui/manga/chapter/ChaptersPresenter.kt

@@ -96,7 +96,7 @@ class ChaptersPresenter : BasePresenter<ChaptersFragment>() {
     }
 
     fun getOnlineChaptersObs(): Observable<Pair<Int, Int>> {
-        return source.pullChaptersFromNetwork(manga.url)
+        return source.fetchChapterList(manga)
                 .subscribeOn(Schedulers.io())
                 .map { syncChaptersWithSource(db, it, manga, source) }
                 .observeOn(AndroidSchedulers.mainThread())

app/src/main/java/eu/kanade/tachiyomi/ui/manga/info/MangaInfoFragment.kt

@@ -8,6 +8,7 @@ import com.bumptech.glide.Glide
 import com.bumptech.glide.load.engine.DiskCacheStrategy
 import eu.kanade.tachiyomi.R
 import eu.kanade.tachiyomi.data.database.models.Manga
+import eu.kanade.tachiyomi.data.source.base.OnlineSource
 import eu.kanade.tachiyomi.data.source.base.Source
 import eu.kanade.tachiyomi.ui.base.fragment.BaseRxFragment
 import eu.kanade.tachiyomi.util.getResourceColor
@@ -96,7 +97,7 @@ class MangaInfoFragment : BaseRxFragment<MangaInfoPresenter>() {
 
         // If manga source is known update source TextView.
         if (source != null) {
-            manga_source.text = source.visibleName
+            manga_source.text = source.toString()
         }
 
         // Update genres TextView.
@@ -140,8 +141,9 @@ class MangaInfoFragment : BaseRxFragment<MangaInfoPresenter>() {
      * Open the manga in browser.
      */
     fun openInBrowser() {
+        val source = presenter.source as? OnlineSource ?: return
         try {
-            val url = Uri.parse(presenter.source.baseUrl + presenter.manga.url)
+            val url = Uri.parse(source.baseUrl + presenter.manga.url)
             val intent = CustomTabsIntent.Builder()
                     .setToolbarColor(context.theme.getResourceColor(R.attr.colorPrimary))
                     .build()

app/src/main/java/eu/kanade/tachiyomi/ui/manga/info/MangaInfoPresenter.kt

@@ -99,7 +99,7 @@ class MangaInfoPresenter : BasePresenter<MangaInfoFragment>() {
      * @return manga information.
      */
     private fun fetchMangaObs(): Observable<Manga> {
-        return source.pullMangaFromNetwork(manga.url)
+        return source.fetchMangaDetails(manga)
                 .flatMap { networkManga ->
                     manga.copyFrom(networkManga)
                     db.insertManga(manga).executeAsBlocking()

app/src/main/java/eu/kanade/tachiyomi/ui/reader/ReaderPresenter.kt

@@ -12,6 +12,7 @@ import eu.kanade.tachiyomi.data.mangasync.MangaSyncManager
 import eu.kanade.tachiyomi.data.mangasync.UpdateMangaSyncService
 import eu.kanade.tachiyomi.data.preference.PreferencesHelper
 import eu.kanade.tachiyomi.data.source.SourceManager
+import eu.kanade.tachiyomi.data.source.base.OnlineSource
 import eu.kanade.tachiyomi.data.source.base.Source
 import eu.kanade.tachiyomi.data.source.model.Page
 import eu.kanade.tachiyomi.ui.base.presenter.BasePresenter
@@ -126,9 +127,16 @@ class ReaderPresenter : BasePresenter<ReaderActivity>() {
                         observable = Observable.from(ch.pages)
                                 .flatMap { downloadManager.getDownloadedImage(it, chapterDir) }
                     } else {
-                        observable = source.getAllImageUrlsFromPageList(ch.pages)
-                                .flatMap({ source.getCachedImage(it) }, 2)
-                                .doOnCompleted { source.savePageList(ch.url, ch.pages) }
+                        observable = source.let { source ->
+                            if (source is OnlineSource) {
+                                source.fetchAllImageUrlsFromPageList(ch.pages)
+                                        .flatMap({ source.getCachedImage(it) }, 2)
+                                        .doOnCompleted { source.savePageList(ch, ch.pages) }
+                            } else {
+                                Observable.from(ch.pages)
+                                        .flatMap { source.fetchImage(it) }
+                            }
+                        }
                     }
                     observable.doOnCompleted {
                         if (!isSeamlessMode && chapter === ch) {
@@ -139,13 +147,7 @@ class ReaderPresenter : BasePresenter<ReaderActivity>() {
 
         // Listen por retry events
         add(retryPageSubject.observeOn(Schedulers.io())
-                .flatMap { page ->
-                    if (page.imageUrl == null)
-                        source.getImageUrlFromPage(page)
-                    else
-                        Observable.just<Page>(page)
-                }
-                .flatMap { source.getCachedImage(it) }
+                .flatMap { source.fetchImage(it) }
                 .subscribe())
     }
 
@@ -156,7 +158,7 @@ class ReaderPresenter : BasePresenter<ReaderActivity>() {
             Observable.just(downloadManager.getSavedPageList(source, manga, chapter)!!)
         else
         // Fetch the page list from cache or fallback to network
-            source.getCachedPageListOrPullFromNetwork(chapter.url)
+            source.fetchPageList(chapter)
                     .subscribeOn(Schedulers.io())
                     .observeOn(AndroidSchedulers.mainThread())
 
@@ -200,26 +202,15 @@ class ReaderPresenter : BasePresenter<ReaderActivity>() {
 
     // Preload the first pages of the next chapter. Only for non seamless mode
     private fun getPreloadNextChapterObservable(): Observable<Page> {
-        return source.getCachedPageListOrPullFromNetwork(nextChapter!!.url)
+        val nextChapter = nextChapter ?: return Observable.error(Exception("No next chapter"))
+        return source.fetchPageList(nextChapter)
                 .flatMap { pages ->
-                    nextChapter!!.pages = pages
+                    nextChapter.pages = pages
                     val pagesToPreload = Math.min(pages.size, 5)
                     Observable.from(pages).take(pagesToPreload)
                 }
                 // Preload up to 5 images
-                .concatMap { page ->
-                    if (page.imageUrl == null)
-                        source.getImageUrlFromPage(page)
-                    else
-                        Observable.just<Page>(page)
-                }
-                // Download the first image
-                .concatMap { page ->
-                    if (page.pageNumber == 0)
-                        source.getCachedImage(page)
-                    else
-                        Observable.just<Page>(page)
-                }
+                .concatMap { source.fetchImage(it) }
                 .subscribeOn(Schedulers.io())
                 .observeOn(AndroidSchedulers.mainThread())
                 .doOnCompleted { stopPreloadingNextChapter() }
@@ -324,7 +315,7 @@ class ReaderPresenter : BasePresenter<ReaderActivity>() {
 
         // Cache current page list progress for online chapters to allow a faster reopen
         if (!chapter.isDownloaded) {
-            source.savePageList(chapter.url, pages)
+            source.let { if (it is OnlineSource) it.savePageList(chapter, pages) }
         }
 
         // Save current progress of the chapter. Mark as read if the chapter is finished
@@ -382,7 +373,7 @@ class ReaderPresenter : BasePresenter<ReaderActivity>() {
     }
 
     fun updateMangaSyncLastChapterRead() {
-        for (mangaSync in mangaSyncList!!) {
+        for (mangaSync in mangaSyncList ?: emptyList()) {
             val service = syncManager.getService(mangaSync.sync_id)
             if (service.isLogged && mangaSync.update) {
                 UpdateMangaSyncService.start(context, mangaSync)
@@ -417,16 +408,21 @@ class ReaderPresenter : BasePresenter<ReaderActivity>() {
     }
 
     private fun preloadNextChapter() {
-        if (hasNextChapter() && !isChapterDownloaded(nextChapter!!)) {
-            start(PRELOAD_NEXT_CHAPTER)
+        nextChapter?.let {
+            if (!isChapterDownloaded(it)) {
+                start(PRELOAD_NEXT_CHAPTER)
+            }
         }
     }
 
     private fun stopPreloadingNextChapter() {
         if (!isUnsubscribed(PRELOAD_NEXT_CHAPTER)) {
             stop(PRELOAD_NEXT_CHAPTER)
-            if (nextChapter!!.pages != null)
-                source.savePageList(nextChapter!!.url, nextChapter!!.pages)
+            nextChapter?.let { chapter ->
+                if (chapter.pages != null) {
+                    source.let { if (it is OnlineSource) it.savePageList(chapter, chapter.pages) }
+                }
+            }
         }
     }
 

app/src/main/java/eu/kanade/tachiyomi/ui/setting/SettingsSourcesFragment.kt

@@ -42,11 +42,11 @@ class SettingsSourcesFragment : SettingsNestedFragment() {
                 .subscribe { languages ->
                     sourcesPref.removeAll()
 
-                    val enabledSources = settingsActivity.sourceManager.getSources()
+                    val enabledSources = settingsActivity.sourceManager.getOnlineSources()
                             .filter { it.lang.code in languages }
 
                     for (source in enabledSources) {
-                        if (source.isLoginRequired) {
+                        if (source.isLoginRequired()) {
                             val pref = createSource(source)
                             sourcesPref.addPreference(pref)
                         }
@@ -65,7 +65,7 @@ class SettingsSourcesFragment : SettingsNestedFragment() {
     fun createSource(source: Source): Preference {
         return LoginPreference(preferenceManager.context).apply {
             key = preferences.keys.sourceUsername(source.id)
-            title = source.visibleName
+            title = source.toString()
 
             setOnPreferenceClickListener {
                 val fragment = SourceLoginDialog.newInstance(source)

app/src/main/java/eu/kanade/tachiyomi/util/ChapterSourceSync.kt

@@ -3,6 +3,7 @@ package eu.kanade.tachiyomi.util
 import eu.kanade.tachiyomi.data.database.DatabaseHelper
 import eu.kanade.tachiyomi.data.database.models.Chapter
 import eu.kanade.tachiyomi.data.database.models.Manga
+import eu.kanade.tachiyomi.data.source.base.OnlineSource
 import eu.kanade.tachiyomi.data.source.base.Source
 import java.util.*
 
@@ -34,7 +35,9 @@ fun syncChaptersWithSource(db: DatabaseHelper,
 
     // Recognize number for new chapters.
     toAdd.forEach {
-        source.parseChapterNumber(it)
+        if (source is OnlineSource) {
+            source.parseChapterNumber(it)
+        }
         ChapterRecognition.parseChapterNumber(it, manga)
     }
 

app/src/main/java/eu/kanade/tachiyomi/util/Parser.java

@@ -1,52 +0,0 @@
-package eu.kanade.tachiyomi.util;
-
-import android.support.annotation.Nullable;
-
-import org.jsoup.nodes.Element;
-import org.jsoup.select.Elements;
-
-public final class Parser {
-
-    private Parser() throws InstantiationException {
-        throw new InstantiationException("This class is not for instantiation");
-    }
-
-    @Nullable
-    public static Element element(Element container, String pattern) {
-        return container.select(pattern).first();
-    }
-
-    @Nullable
-    public static String text(Element container, String pattern) {
-        return text(container, pattern, null);
-    }
-
-    @Nullable
-    public static String text(Element container, String pattern, String defValue) {
-        Element element = container.select(pattern).first();
-        return element != null ? element.text() : defValue;
-    }
-
-    @Nullable
-    public static String allText(Element container, String pattern) {
-        Elements elements = container.select(pattern);
-        return !elements.isEmpty() ? elements.text() : null;
-    }
-
-    @Nullable
-    public static String attr(Element container, String pattern, String attr) {
-        Element element = container.select(pattern).first();
-        return element != null ? element.attr(attr) : null;
-    }
-
-    @Nullable
-    public static String href(Element container, String pattern) {
-        return attr(container, pattern, "href");
-    }
-
-    @Nullable
-    public static String src(Element container, String pattern) {
-        return attr(container, pattern, "src");
-    }
-
-}

app/src/main/java/eu/kanade/tachiyomi/widget/preference/SourceLoginDialog.kt

@@ -3,6 +3,7 @@ package eu.kanade.tachiyomi.widget.preference
 import android.os.Bundle
 import android.view.View
 import eu.kanade.tachiyomi.R
+import eu.kanade.tachiyomi.data.source.base.OnlineSource
 import eu.kanade.tachiyomi.data.source.base.Source
 import eu.kanade.tachiyomi.ui.setting.SettingsActivity
 import eu.kanade.tachiyomi.util.toast
@@ -23,17 +24,17 @@ class SourceLoginDialog : LoginDialogPreference() {
         }
     }
 
-    lateinit var source: Source
+    lateinit var source: OnlineSource
 
     override fun onCreate(savedInstanceState: Bundle?) {
         super.onCreate(savedInstanceState)
 
         val sourceId = arguments.getInt("key")
-        source = (activity as SettingsActivity).sourceManager.get(sourceId)!!
+        source = (activity as SettingsActivity).sourceManager.get(sourceId) as OnlineSource
     }
 
     override fun setCredentialsOnView(view: View) = with(view) {
-        dialog_title.text = getString(R.string.login_title, source.visibleName)
+        dialog_title.text = getString(R.string.login_title, source.toString())
         username.setText(preferences.sourceUsername(source))
         password.setText(preferences.sourcePassword(source))
     }

app/src/test/java/eu/kanade/tachiyomi/data/library/LibraryUpdateServiceTest.java

@@ -17,7 +17,7 @@ import eu.kanade.tachiyomi.BuildConfig;
 import eu.kanade.tachiyomi.CustomRobolectricGradleTestRunner;
 import eu.kanade.tachiyomi.data.database.models.Chapter;
 import eu.kanade.tachiyomi.data.database.models.Manga;
-import eu.kanade.tachiyomi.data.source.base.Source;
+import eu.kanade.tachiyomi.data.source.base.OnlineSource;
 import rx.Observable;
 
 import static org.assertj.core.api.Assertions.assertThat;
@@ -32,14 +32,14 @@ public class LibraryUpdateServiceTest {
     ShadowApplication app;
     Context context;
     LibraryUpdateService service;
-    Source source;
+    OnlineSource source;
 
     @Before
     public void setup() {
         app = ShadowApplication.getInstance();
         context = app.getApplicationContext();
         service = Robolectric.setupService(LibraryUpdateService.class);
-        source = mock(Source.class);
+        source = mock(OnlineSource.class);
         when(service.sourceManager.get(anyInt())).thenReturn(source);
     }
 
@@ -62,7 +62,7 @@ public class LibraryUpdateServiceTest {
 
         List<Chapter> sourceChapters = createChapters("/chapter1", "/chapter2");
 
-        when(source.pullChaptersFromNetwork(manga.url)).thenReturn(Observable.just(sourceChapters));
+        when(source.fetchChapterList(manga)).thenReturn(Observable.just(sourceChapters));
 
         service.updateManga(manga).subscribe();
 
@@ -79,9 +79,9 @@ public class LibraryUpdateServiceTest {
         List<Chapter> chapters3 = createChapters("/achapter1", "/achapter2");
 
         // One of the updates will fail
-        when(source.pullChaptersFromNetwork("/manga1")).thenReturn(Observable.just(chapters));
-        when(source.pullChaptersFromNetwork("/manga2")).thenReturn(Observable.<List<Chapter>>error(new Exception()));
-        when(source.pullChaptersFromNetwork("/manga3")).thenReturn(Observable.just(chapters3));
+        when(source.fetchChapterList(favManga.get(0))).thenReturn(Observable.just(chapters));
+        when(source.fetchChapterList(favManga.get(1))).thenReturn(Observable.<List<Chapter>>error(new Exception()));
+        when(source.fetchChapterList(favManga.get(2))).thenReturn(Observable.just(chapters3));
 
         service.updateMangaList(service.getMangaToUpdate(null)).subscribe();