|
@@ -10,11 +10,49 @@ import eu.kanade.mangafeed.data.caches.CacheManager;
|
|
|
import eu.kanade.mangafeed.data.helpers.NetworkHelper;
|
|
|
import eu.kanade.mangafeed.data.models.Chapter;
|
|
|
import eu.kanade.mangafeed.data.models.Manga;
|
|
|
+import eu.kanade.mangafeed.data.models.Page;
|
|
|
import rx.Observable;
|
|
|
import rx.schedulers.Schedulers;
|
|
|
|
|
|
public abstract class Source {
|
|
|
|
|
|
+ // Methods to implement or optionally override
|
|
|
+
|
|
|
+ // Name of the source to display
|
|
|
+ public abstract String getName();
|
|
|
+
|
|
|
+ // Id of the source (must be declared and obtained from SourceManager to avoid conflicts)
|
|
|
+ public abstract int getSourceId();
|
|
|
+
|
|
|
+ protected abstract String getUrlFromPageNumber(int page);
|
|
|
+ protected abstract String getSearchUrl(String query, int page);
|
|
|
+ protected abstract List<Manga> parsePopularMangasFromHtml(String unparsedHtml);
|
|
|
+ protected abstract List<Manga> parseSearchFromHtml(String unparsedHtml);
|
|
|
+ protected abstract Manga parseHtmlToManga(String mangaUrl, String unparsedHtml);
|
|
|
+ protected abstract List<Chapter> parseHtmlToChapters(String unparsedHtml);
|
|
|
+ protected abstract List<String> parseHtmlToPageUrls(String unparsedHtml);
|
|
|
+ protected abstract String parseHtmlToImageUrl(String unparsedHtml);
|
|
|
+
|
|
|
+ // Get the URL to the details of a manga, useful if the source provides some kind of API or fast calls
|
|
|
+ protected String getMangaUrl(String defaultMangaUrl) {
|
|
|
+ return defaultMangaUrl;
|
|
|
+ }
|
|
|
+
|
|
|
+ // Default headers, it can be overriden by children or just add new keys
|
|
|
+ protected Headers.Builder headersBuilder() {
|
|
|
+ Headers.Builder builder = new Headers.Builder();
|
|
|
+ builder.add("User-Agent", "Mozilla/5.0 (Windows NT 6.3; WOW64)");
|
|
|
+ return builder;
|
|
|
+ }
|
|
|
+
|
|
|
+ // Number of images to download at the same time
|
|
|
+ protected int getNumberOfConcurrentImageDownloads() {
|
|
|
+ return 3;
|
|
|
+ }
|
|
|
+
|
|
|
+
|
|
|
+ // ***** Source class implementation *****
|
|
|
+
|
|
|
protected NetworkHelper mNetworkService;
|
|
|
protected CacheManager mCacheManager;
|
|
|
protected Headers mRequestHeaders;
|
|
@@ -25,13 +63,6 @@ public abstract class Source {
|
|
|
mRequestHeaders = headersBuilder().build();
|
|
|
}
|
|
|
|
|
|
- // Default headers, it can be overriden by children or add new keys
|
|
|
- protected Headers.Builder headersBuilder() {
|
|
|
- Headers.Builder builder = new Headers.Builder();
|
|
|
- builder.add("User-Agent", "Mozilla/5.0 (Windows NT 6.3; WOW64)");
|
|
|
- return builder;
|
|
|
- }
|
|
|
-
|
|
|
// Get the most popular mangas from the source
|
|
|
public Observable<List<Manga>> pullPopularMangasFromNetwork(int page) {
|
|
|
String url = getUrlFromPageNumber(page);
|
|
@@ -62,56 +93,54 @@ public abstract class Source {
|
|
|
Observable.just(parseHtmlToChapters(unparsedHtml)));
|
|
|
}
|
|
|
|
|
|
+ public Observable<List<Page>> pullPageListFromNetwork(final String chapterUrl) {
|
|
|
+ return mCacheManager.getPageUrlsFromDiskCache(chapterUrl)
|
|
|
+ .onErrorResumeNext(throwable -> {
|
|
|
+ return mNetworkService
|
|
|
+ .getStringResponse(chapterUrl, mNetworkService.NULL_CACHE_CONTROL, mRequestHeaders)
|
|
|
+ .flatMap(unparsedHtml -> Observable.just(parseHtmlToPageUrls(unparsedHtml)))
|
|
|
+ .flatMap(this::convertToPages)
|
|
|
+ .doOnNext(pages -> savePageList(chapterUrl, pages));
|
|
|
+ })
|
|
|
+ .onBackpressureBuffer();
|
|
|
+ }
|
|
|
+
|
|
|
// Get the URLs of the images of a chapter
|
|
|
- public Observable<String> getImageUrlsFromNetwork(final String chapterUrl) {
|
|
|
- return mNetworkService
|
|
|
- .getStringResponse(chapterUrl, mNetworkService.NULL_CACHE_CONTROL, mRequestHeaders)
|
|
|
- .flatMap(unparsedHtml -> Observable.from(parseHtmlToPageUrls(unparsedHtml)))
|
|
|
- .buffer(3)
|
|
|
- .concatMap(batchedPageUrls -> {
|
|
|
- List<Observable<String>> imageUrlObservables = new ArrayList<>();
|
|
|
- for (String pageUrl : batchedPageUrls) {
|
|
|
- Observable<String> temporaryObservable = mNetworkService
|
|
|
- .getStringResponse(pageUrl, mNetworkService.NULL_CACHE_CONTROL, mRequestHeaders)
|
|
|
- .flatMap(unparsedHtml -> Observable.just(parseHtmlToImageUrl(unparsedHtml)))
|
|
|
- .subscribeOn(Schedulers.io());
|
|
|
-
|
|
|
- imageUrlObservables.add(temporaryObservable);
|
|
|
+ public Observable<Page> getRemainingImageUrlsFromPageList(final List<Page> pages) {
|
|
|
+ return Observable.from(pages)
|
|
|
+ .filter(page -> page.getImageUrl() == null)
|
|
|
+ .buffer(getNumberOfConcurrentImageDownloads())
|
|
|
+ .concatMap(batchedPages -> {
|
|
|
+ List<Observable<Page>> pageObservable = new ArrayList<>();
|
|
|
+ for (Page page : batchedPages) {
|
|
|
+ pageObservable.add(getImageUrlFromPage(page));
|
|
|
}
|
|
|
-
|
|
|
- return Observable.merge(imageUrlObservables);
|
|
|
+ return Observable.merge(pageObservable);
|
|
|
});
|
|
|
}
|
|
|
|
|
|
- // Store the URLs of a chapter in the cache
|
|
|
- public Observable<String> pullImageUrlsFromNetwork(final String chapterUrl) {
|
|
|
- final List<String> temporaryCachedImageUrls = new ArrayList<>();
|
|
|
-
|
|
|
- return mCacheManager.getImageUrlsFromDiskCache(chapterUrl)
|
|
|
- .onErrorResumeNext(throwable -> {
|
|
|
- return getImageUrlsFromNetwork(chapterUrl)
|
|
|
- .doOnNext(imageUrl -> temporaryCachedImageUrls.add(imageUrl))
|
|
|
- .doOnCompleted(mCacheManager.putImageUrlsToDiskCache(chapterUrl, temporaryCachedImageUrls));
|
|
|
+ private Observable<Page> getImageUrlFromPage(final Page page) {
|
|
|
+ return mNetworkService
|
|
|
+ .getStringResponse(page.getUrl(), mNetworkService.NULL_CACHE_CONTROL, mRequestHeaders)
|
|
|
+ .flatMap(unparsedHtml -> Observable.just(parseHtmlToImageUrl(unparsedHtml)))
|
|
|
+ .flatMap(imageUrl -> {
|
|
|
+ page.setImageUrl(imageUrl);
|
|
|
+ return Observable.just(page);
|
|
|
})
|
|
|
- .onBackpressureBuffer();
|
|
|
+ .subscribeOn(Schedulers.io());
|
|
|
}
|
|
|
|
|
|
- // Get the URL to the details of a manga, useful if the source provides some kind of API or fast calls
|
|
|
- protected String getMangaUrl(String defaultMangaUrl) {
|
|
|
- return defaultMangaUrl;
|
|
|
+ public void savePageList(String chapterUrl, List<Page> pages) {
|
|
|
+ mCacheManager.putPageUrlsToDiskCache(chapterUrl, pages);
|
|
|
}
|
|
|
|
|
|
- public abstract String getName();
|
|
|
- public abstract int getSourceId();
|
|
|
-
|
|
|
- protected abstract String getUrlFromPageNumber(int page);
|
|
|
- protected abstract String getSearchUrl(String query, int page);
|
|
|
- protected abstract List<Manga> parsePopularMangasFromHtml(String unparsedHtml);
|
|
|
- protected abstract List<Manga> parseSearchFromHtml(String unparsedHtml);
|
|
|
- protected abstract Manga parseHtmlToManga(String mangaUrl, String unparsedHtml);
|
|
|
- protected abstract List<Chapter> parseHtmlToChapters(String unparsedHtml);
|
|
|
- protected abstract List<String> parseHtmlToPageUrls(String unparsedHtml);
|
|
|
- protected abstract String parseHtmlToImageUrl(String unparsedHtml);
|
|
|
+ private Observable<List<Page>> convertToPages(List<String> pageUrls) {
|
|
|
+ List<Page> pages = new ArrayList<>();
|
|
|
+ for (int i = 0; i < pageUrls.size(); i++) {
|
|
|
+ pages.add(new Page(i, pageUrls.get(i)));
|
|
|
+ }
|
|
|
+ return Observable.just(pages);
|
|
|
+ }
|
|
|
|
|
|
|
|
|
}
|