From 75f040e77a3330667b60d02689998d77b9c2fb1e Mon Sep 17 00:00:00 2001 From: az4521 <18432684+az4521@users.noreply.github.com> Date: Sat, 9 Mar 2024 06:24:52 +0000 Subject: [PATCH] remove 8muses, add migration --- .../tachiyomi/extension/ExtensionManager.kt | 2 - .../kanade/tachiyomi/source/SourceManager.kt | 2 - .../source/online/english/EightMuses.kt | 399 ------------------ .../kanade/tachiyomi/ui/main/MainActivity.kt | 4 - .../ui/setting/SettingsAdvancedController.kt | 7 - app/src/main/java/exh/EHSourceHelpers.kt | 6 +- app/src/main/java/exh/EXHMigrations.kt | 19 + .../metadata/EightMusesSearchMetadata.kt | 50 --- 8 files changed, 23 insertions(+), 466 deletions(-) delete mode 100644 app/src/main/java/eu/kanade/tachiyomi/source/online/english/EightMuses.kt delete mode 100755 app/src/main/java/exh/metadata/metadata/EightMusesSearchMetadata.kt diff --git a/app/src/main/java/eu/kanade/tachiyomi/extension/ExtensionManager.kt b/app/src/main/java/eu/kanade/tachiyomi/extension/ExtensionManager.kt index 72289fa5f0a7..e00dc28c15d9 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/extension/ExtensionManager.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/extension/ExtensionManager.kt @@ -18,7 +18,6 @@ import eu.kanade.tachiyomi.source.SourceManager import eu.kanade.tachiyomi.util.lang.launchNow import eu.kanade.tachiyomi.util.system.toast import exh.EH_SOURCE_ID -import exh.EIGHTMUSES_SOURCE_ID import exh.EXH_SOURCE_ID import exh.HITOMI_SOURCE_ID import exh.MERGED_SOURCE_ID @@ -81,7 +80,6 @@ class ExtensionManager( EXH_SOURCE_ID -> context.getDrawable(R.mipmap.ic_exhentai_source) NHENTAI_SOURCE_ID -> context.getDrawable(R.mipmap.ic_nhentai_source) HITOMI_SOURCE_ID -> context.getDrawable(R.mipmap.ic_hitomi_source) - EIGHTMUSES_SOURCE_ID -> context.getDrawable(R.mipmap.ic_8muses_source) MERGED_SOURCE_ID -> context.getDrawable(R.mipmap.ic_merged_source) else -> null } diff --git a/app/src/main/java/eu/kanade/tachiyomi/source/SourceManager.kt b/app/src/main/java/eu/kanade/tachiyomi/source/SourceManager.kt index d2faf186ce9c..a4456e0eb6e0 100755 --- a/app/src/main/java/eu/kanade/tachiyomi/source/SourceManager.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/source/SourceManager.kt @@ -13,7 +13,6 @@ import eu.kanade.tachiyomi.source.online.all.EHentai import eu.kanade.tachiyomi.source.online.all.Hitomi import eu.kanade.tachiyomi.source.online.all.MergedSource import eu.kanade.tachiyomi.source.online.all.NHentai -import eu.kanade.tachiyomi.source.online.english.EightMuses import eu.kanade.tachiyomi.source.online.english.HentaiCafe import eu.kanade.tachiyomi.source.online.english.Pururin import eu.kanade.tachiyomi.source.online.english.Tsumino @@ -117,7 +116,6 @@ open class SourceManager(private val context: Context) { } exSrcs += NHentai(context) exSrcs += Hitomi() - exSrcs += EightMuses() return exSrcs } diff --git a/app/src/main/java/eu/kanade/tachiyomi/source/online/english/EightMuses.kt b/app/src/main/java/eu/kanade/tachiyomi/source/online/english/EightMuses.kt deleted file mode 100644 index 8306666b341b..000000000000 --- a/app/src/main/java/eu/kanade/tachiyomi/source/online/english/EightMuses.kt +++ /dev/null @@ -1,399 +0,0 @@ -package eu.kanade.tachiyomi.source.online.english - -import android.net.Uri -import com.kizitonwose.time.hours -import eu.kanade.tachiyomi.network.GET -import eu.kanade.tachiyomi.network.asObservableSuccess -import eu.kanade.tachiyomi.source.model.Filter -import eu.kanade.tachiyomi.source.model.FilterList -import eu.kanade.tachiyomi.source.model.MangasPage -import eu.kanade.tachiyomi.source.model.Page -import eu.kanade.tachiyomi.source.model.SChapter -import eu.kanade.tachiyomi.source.model.SManga -import eu.kanade.tachiyomi.source.online.HttpSource -import eu.kanade.tachiyomi.source.online.LewdSource -import eu.kanade.tachiyomi.source.online.UrlImportableSource -import eu.kanade.tachiyomi.util.asJsoup -import exh.EIGHTMUSES_SOURCE_ID -import exh.metadata.metadata.EightMusesSearchMetadata -import exh.metadata.metadata.base.RaisedTag -import exh.util.CachedField -import exh.util.NakedTrie -import exh.util.await -import exh.util.urlImportFetchSearchManga -import hu.akarnokd.rxjava.interop.RxJavaInterop -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.GlobalScope -import kotlinx.coroutines.async -import kotlinx.coroutines.rx2.asSingle -import kotlinx.coroutines.withContext -import okhttp3.Headers -import okhttp3.HttpUrl -import okhttp3.HttpUrl.Companion.toHttpUrlOrNull -import okhttp3.OkHttpClient -import okhttp3.Request -import okhttp3.Response -import org.jsoup.Jsoup -import org.jsoup.nodes.Document -import org.jsoup.nodes.Element -import rx.Observable -import rx.schedulers.Schedulers - -typealias SiteMap = NakedTrie - -class EightMuses : - HttpSource(), - LewdSource, - UrlImportableSource { - override val id = EIGHTMUSES_SOURCE_ID - - /** - * Name of the source. - */ - override val name = "8muses" - /** - * Whether the source has support for latest updates. - */ - override val supportsLatest = true - /** - * An ISO 639-1 compliant language code (two letters in lower case). - */ - override val lang: String = "en" - - override val metaClass = EightMusesSearchMetadata::class - - /** - * Base url of the website without the trailing slash, like: http://mysite.com - */ - override val baseUrl = EightMusesSearchMetadata.BASE_URL - - private val siteMapCache = CachedField(1.hours.inMilliseconds.longValue) - - override val client: OkHttpClient - get() = network.cloudflareClient - - private suspend fun obtainSiteMap() = siteMapCache.obtain { - withContext(Dispatchers.IO) { - val result = client.newCall(eightMusesGet("$baseUrl/sitemap/1.xml")) - .asObservableSuccess() - .toSingle() - .await(Schedulers.io()) - .body!!.string() - - val parsed = Jsoup.parse(result) - - val seen = NakedTrie() - - parsed.getElementsByTag("loc").forEach { item -> - seen[item.text().substring(22)] = Unit - } - - seen - } - } - - override fun headersBuilder(): Headers.Builder { - return Headers.Builder() - .add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;") - .add("Accept-Language", "en-GB,en-US;q=0.9,en;q=0.8") - .add("Referer", "https://www.8muses.com") - .add("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36") - } - - private fun eightMusesGet(url: String): Request { - return GET(url, headers = headersBuilder().build()) - } - - /** - * Returns the request for the popular manga given the page. - * - * @param page the page number to retrieve. - */ - override fun popularMangaRequest(page: Int) = eightMusesGet("$baseUrl/comics/$page") - - /** - * Parses the response from the site and returns a [MangasPage] object. - * - * @param response the response from the site. - */ - override fun popularMangaParse(response: Response): MangasPage { - throw UnsupportedOperationException("Should not be called!") - } - - /** - * Returns the request for the search manga given the page. - * - * @param page the page number to retrieve. - * @param query the search query. - * @param filters the list of filters to apply. - */ - override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request { - val urlBuilder = if (!query.isBlank()) { - "$baseUrl/search".toHttpUrlOrNull()!! - .newBuilder() - .addQueryParameter("q", query) - } else { - "$baseUrl/comics".toHttpUrlOrNull()!! - .newBuilder() - } - - urlBuilder.addQueryParameter("page", page.toString()) - - filters.filterIsInstance().map { - it.addToUri(urlBuilder) - } - - return eightMusesGet(urlBuilder.toString()) - } - - /** - * Parses the response from the site and returns a [MangasPage] object. - * - * @param response the response from the site. - */ - override fun searchMangaParse(response: Response): MangasPage { - throw UnsupportedOperationException("Should not be called!") - } - - /** - * Returns the request for latest manga given the page. - * - * @param page the page number to retrieve. - */ - override fun latestUpdatesRequest(page: Int) = eightMusesGet("$baseUrl/comics/lastupdate?page=$page") - - /** - * Parses the response from the site and returns a [MangasPage] object. - * - * @param response the response from the site. - */ - override fun latestUpdatesParse(response: Response): MangasPage { - throw UnsupportedOperationException("Should not be called!") - } - -// override fun fetchLatestUpdates(page: Int) = fetchListing(latestUpdatesRequest(page), false) - override fun fetchLatestUpdates(page: Int) = fetchListing(popularMangaRequest(page), false) - - override fun fetchPopularManga(page: Int) = fetchListing(popularMangaRequest(page), false) // TODO Dig - - override fun fetchSearchManga(page: Int, query: String, filters: FilterList): Observable { - return urlImportFetchSearchManga(query) { - fetchListing(searchMangaRequest(page, query, filters), false) - } - } - - private fun fetchListing(request: Request, dig: Boolean): Observable { - return client.newCall(request) - .asObservableSuccess() - .flatMapSingle { response -> - RxJavaInterop.toV1Single( - GlobalScope.async(Dispatchers.IO) { - parseResultsPage(response, dig) - }.asSingle(GlobalScope.coroutineContext) - ) - } - } - - private suspend fun parseResultsPage(response: Response, dig: Boolean): MangasPage { - val doc = response.asJsoup() - val contents = parseSelf(doc) - - val onLastPage = doc.selectFirst(".current:nth-last-child(2)") != null - - return MangasPage( - if (dig) { - contents.albums.flatMap { - val href = it.attr("href") - val splitHref = href.split('/') - obtainSiteMap().subMap(href).filter { - it.key.split('/').size - splitHref.size == 1 - }.map { (key, _) -> - SManga.create().apply { - url = key - - title = key.substringAfterLast('/').replace('-', ' ') - } - } - } - } else { - contents.albums.map { - SManga.create().apply { - url = it.attr("href") - - title = it.select(".title-text").text() - - thumbnail_url = baseUrl + it.select(".lazyload").attr("data-src") - } - } - }, - !onLastPage - ) - } - - /** - * Parses the response from the site and returns the details of a manga. - * - * @param response the response from the site. - */ - override fun mangaDetailsParse(response: Response): SManga { - throw UnsupportedOperationException("Should not be called!") - } - - /** - * Returns an observable with the updated details for a manga. Normally it's not needed to - * override this method. - * - * @param manga the manga to be updated. - */ - override fun fetchMangaDetails(manga: SManga): Observable { - return client.newCall(mangaDetailsRequest(manga)) - .asObservableSuccess() - .flatMap { - parseToManga(manga, it.asJsoup()).andThen(Observable.just(manga)) - } - } - - /** - * Parses the response from the site and returns a list of chapters. - * - * @param response the response from the site. - */ - override fun chapterListParse(response: Response): List { - throw UnsupportedOperationException("Should not be called!") - } - - override fun fetchChapterList(manga: SManga): Observable> { - return RxJavaInterop.toV1Single( - GlobalScope.async(Dispatchers.IO) { - fetchAndParseChapterList("", manga.url) - }.asSingle(GlobalScope.coroutineContext) - ).toObservable() - } - - private suspend fun fetchAndParseChapterList(prefix: String, url: String): List { - // Request - val req = eightMusesGet(baseUrl + url) - - return client.newCall(req).asObservableSuccess().toSingle().toBlocking().value().use { response -> - val contents = parseSelf(response.asJsoup()) - - val out = mutableListOf() - if (contents.images.isNotEmpty()) { - out += SChapter.create().apply { - this.url = url - this.name = if (prefix.isBlank()) ">" else prefix - } - } - - val builtPrefix = if (prefix.isBlank()) "> " else "$prefix > " - - out + contents.albums.flatMap { ele -> - fetchAndParseChapterList(builtPrefix + ele.selectFirst(".title-text")!!.text(), ele.attr("href")) - } - } - } - - data class SelfContents(val albums: List, val images: List) - - private fun parseSelf(doc: Document): SelfContents { - // Parse self - val gc = doc.select(".gallery .c-tile") - - // Check if any in self - val selfAlbums = gc.filter { it.attr("href").startsWith("/comics/album") } - val selfImages = gc.filter { it.attr("href").startsWith("/comics/picture") } - - return SelfContents(selfAlbums, selfImages) - } - - /** - * Parses the response from the site and returns a list of pages. - * - * @param response the response from the site. - */ - override fun pageListParse(response: Response): List { - val contents = parseSelf(response.asJsoup()) - return contents.images.mapIndexed { index, element -> - Page( - index, - element.attr("href"), - "$baseUrl/image/fl" + element.select(".lazyload").attr("data-src").substring(9) - ) - } - } - - override fun parseIntoMetadata(metadata: EightMusesSearchMetadata, input: Document) { - with(metadata) { - path = Uri.parse(input.location()).pathSegments - - val breadcrumbs = input.selectFirst(".top-menu-breadcrumb > ol") - - title = breadcrumbs!!.selectFirst("li:nth-last-child(1) > a")!!.text() - - thumbnailUrl = parseSelf(input).let { it.albums + it.images }.firstOrNull() - ?.selectFirst(".lazyload") - ?.attr("data-src")?.let { - baseUrl + it - } - - tags.clear() - tags += RaisedTag( - EightMusesSearchMetadata.ARTIST_NAMESPACE, - breadcrumbs!!.selectFirst("li:nth-child(2) > a")!!.text(), - EightMusesSearchMetadata.TAG_TYPE_DEFAULT - ) - tags += input.select(".album-tags a").map { - RaisedTag( - EightMusesSearchMetadata.TAGS_NAMESPACE, - it.text(), - EightMusesSearchMetadata.TAG_TYPE_DEFAULT - ) - } - } - } - - class SortFilter : Filter.Select( - "Sort", - SORT_OPTIONS.map { it.second }.toTypedArray() - ) { - fun addToUri(url: HttpUrl.Builder) { - url.addQueryParameter("sort", SORT_OPTIONS[state].first) - } - - companion object { - // - private val SORT_OPTIONS = listOf( - "" to "Views", - "like" to "Likes", - "date" to "Date", - "az" to "A-Z" - ) - } - } - - override fun getFilterList() = FilterList( - SortFilter() - ) - - /** - * Parses the response from the site and returns the absolute url to the source image. - * - * @param response the response from the site. - */ - override fun imageUrlParse(response: Response): String { - throw UnsupportedOperationException("Should not be called!") - } - - override val matchingHosts = listOf( - "www.8muses.com", - "8muses.com" - ) - - override fun mapUrlToMangaUrl(uri: Uri): String? { - var path = uri.pathSegments.drop(2) - if (uri.pathSegments[1].toLowerCase() == "picture") { - path = path.dropLast(1) - } - return "/comics/album/${path.joinToString("/")}" - } -} diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/main/MainActivity.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/main/MainActivity.kt index 2d0131265463..a79b420dd893 100755 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/main/MainActivity.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/main/MainActivity.kt @@ -49,7 +49,6 @@ import eu.kanade.tachiyomi.util.system.vibrate import eu.kanade.tachiyomi.util.view.gone import eu.kanade.tachiyomi.util.view.visible import exh.EH_SOURCE_ID -import exh.EIGHTMUSES_SOURCE_ID import exh.EXHMigrations import exh.EXH_SOURCE_ID import exh.HITOMI_SOURCE_ID @@ -282,9 +281,6 @@ class MainActivity : BaseActivity() { if (HITOMI_SOURCE_ID !in BlacklistedSources.HIDDEN_SOURCES) { BlacklistedSources.HIDDEN_SOURCES += HITOMI_SOURCE_ID } - if (EIGHTMUSES_SOURCE_ID !in BlacklistedSources.HIDDEN_SOURCES) { - BlacklistedSources.HIDDEN_SOURCES += EIGHTMUSES_SOURCE_ID - } } // EXH <-- diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/setting/SettingsAdvancedController.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/setting/SettingsAdvancedController.kt index 918d7d5a47b2..f319995c3249 100755 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/setting/SettingsAdvancedController.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/setting/SettingsAdvancedController.kt @@ -42,7 +42,6 @@ import eu.kanade.tachiyomi.util.system.isPackageInstalled import eu.kanade.tachiyomi.util.system.powerManager import eu.kanade.tachiyomi.util.system.toast import exh.EH_SOURCE_ID -import exh.EIGHTMUSES_SOURCE_ID import exh.EXH_SOURCE_ID import exh.HITOMI_SOURCE_ID import exh.NHENTAI_SOURCE_ID @@ -247,9 +246,6 @@ class SettingsAdvancedController : SettingsController() { if (HITOMI_SOURCE_ID !in BlacklistedSources.HIDDEN_SOURCES) { BlacklistedSources.HIDDEN_SOURCES += HITOMI_SOURCE_ID } - if (EIGHTMUSES_SOURCE_ID !in BlacklistedSources.HIDDEN_SOURCES) { - BlacklistedSources.HIDDEN_SOURCES += EIGHTMUSES_SOURCE_ID - } } else { if (EH_SOURCE_ID in BlacklistedSources.HIDDEN_SOURCES) { BlacklistedSources.HIDDEN_SOURCES -= EH_SOURCE_ID @@ -263,9 +259,6 @@ class SettingsAdvancedController : SettingsController() { if (HITOMI_SOURCE_ID in BlacklistedSources.HIDDEN_SOURCES) { BlacklistedSources.HIDDEN_SOURCES -= HITOMI_SOURCE_ID } - if (EIGHTMUSES_SOURCE_ID in BlacklistedSources.HIDDEN_SOURCES) { - BlacklistedSources.HIDDEN_SOURCES -= EIGHTMUSES_SOURCE_ID - } } true } diff --git a/app/src/main/java/exh/EHSourceHelpers.kt b/app/src/main/java/exh/EHSourceHelpers.kt index a3d02e9311a6..919044ca5ad1 100755 --- a/app/src/main/java/exh/EHSourceHelpers.kt +++ b/app/src/main/java/exh/EHSourceHelpers.kt @@ -19,9 +19,11 @@ val HENTAI_CAFE_SOURCE_ID = delegatedSourceId() val PURURIN_SOURCE_ID = delegatedSourceId() val TSUMINO_SOURCE_ID = delegatedSourceId() const val HITOMI_SOURCE_ID = LEWD_SOURCE_SERIES + 10 -const val EIGHTMUSES_SOURCE_ID = LEWD_SOURCE_SERIES + 11 const val MERGED_SOURCE_ID = LEWD_SOURCE_SERIES + 69 +// Un-Lewded source IDs +const val EIGHTMUSES_SOURCE_ID = 1802675169972965535 + private val DELEGATED_LEWD_SOURCES = listOf( HentaiCafe::class, Pururin::class, @@ -55,4 +57,4 @@ fun isLewdSource(source: Long) = source in 6900..6999 || fun Source.isEhBasedSource() = id == EH_SOURCE_ID || id == EXH_SOURCE_ID -fun Source.isNamespaceSource() = id == EH_SOURCE_ID || id == EXH_SOURCE_ID || id == NHENTAI_SOURCE_ID || id == HITOMI_SOURCE_ID || id == PURURIN_SOURCE_ID || id == TSUMINO_SOURCE_ID || id == EIGHTMUSES_SOURCE_ID +fun Source.isNamespaceSource() = id == EH_SOURCE_ID || id == EXH_SOURCE_ID || id == NHENTAI_SOURCE_ID || id == HITOMI_SOURCE_ID || id == PURURIN_SOURCE_ID || id == TSUMINO_SOURCE_ID diff --git a/app/src/main/java/exh/EXHMigrations.kt b/app/src/main/java/exh/EXHMigrations.kt index 27da7ae86116..642f0e48b2f9 100755 --- a/app/src/main/java/exh/EXHMigrations.kt +++ b/app/src/main/java/exh/EXHMigrations.kt @@ -174,6 +174,25 @@ object EXHMigrations { LibraryUpdateJob.setupTask(context) } } + + if (oldVersion < 8810) { + db.inTransaction { + // Migrate 8Muses source IDs + db.lowLevel().executeSQL( + RawQuery.builder() + .query( + """ + UPDATE ${MangaTable.TABLE} + SET ${MangaTable.COL_SOURCE} = $EIGHTMUSES_SOURCE_ID + WHERE ${MangaTable.COL_SOURCE} = 6911 + """.trimIndent() + ) + .affectsTables(MangaTable.TABLE) + .build() + ) + } + } + // TODO BE CAREFUL TO NOT FUCK UP MergedSources IF CHANGING URLs preferences.eh_lastVersionCode().set(BuildConfig.VERSION_CODE) diff --git a/app/src/main/java/exh/metadata/metadata/EightMusesSearchMetadata.kt b/app/src/main/java/exh/metadata/metadata/EightMusesSearchMetadata.kt deleted file mode 100755 index 62cc42918fd6..000000000000 --- a/app/src/main/java/exh/metadata/metadata/EightMusesSearchMetadata.kt +++ /dev/null @@ -1,50 +0,0 @@ -package exh.metadata.metadata - -import eu.kanade.tachiyomi.source.model.SManga -import exh.plusAssign -import kotlinx.serialization.Serializable - -@Serializable -class EightMusesSearchMetadata : RaisedSearchMetadata() { - var path: List = emptyList() - - var title by titleDelegate(TITLE_TYPE_MAIN) - - var thumbnailUrl: String? = null - - override fun copyTo(manga: SManga) { - manga.url = path.joinToString("/", prefix = "/") - - title?.let { - manga.title = it - } - - thumbnailUrl?.let { - manga.thumbnail_url = it - } - - manga.artist = tags.ofNamespace(ARTIST_NAMESPACE).joinToString { it.name } - - manga.genre = tagsToGenreString() - - val titleDesc = StringBuilder() - title?.let { titleDesc += "Title: $it\n" } - - val tagsDesc = tagsToDescription() - - manga.description = listOf(titleDesc.toString(), tagsDesc.toString()) - .filter(String::isNotBlank) - .joinToString(separator = "\n") - } - - companion object { - private const val TITLE_TYPE_MAIN = 0 - - const val TAG_TYPE_DEFAULT = 0 - - const val BASE_URL = "https://www.8muses.com" - - const val TAGS_NAMESPACE = "tags" - const val ARTIST_NAMESPACE = "artist" - } -}