diff --git a/app/build.gradle b/app/build.gradle index d03bd64e320..4557e18bfeb 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -18,7 +18,7 @@ android { defaultConfig { applicationId "org.schabi.newpipe" resValue "string", "app_name", "NewPipe" - minSdk 21 + minSdk 26 targetSdk 33 if (System.properties.containsKey('versionCodeOverride')) { versionCode System.getProperty('versionCodeOverride') as Integer @@ -108,7 +108,10 @@ android { // no idea how they ended up in the META-INF dir... excludes += ['META-INF/README.md', 'META-INF/CHANGES', // 'COPYRIGHT' belongs to RxJava... - 'META-INF/COPYRIGHT'] + 'META-INF/COPYRIGHT', + // belongs + 'META-INF/DEPENDENCIES' + ] } } } @@ -296,6 +299,10 @@ dependencies { // Date and time formatting implementation "org.ocpsoft.prettytime:prettytime:5.0.8.Final" + // Web browser automation + implementation "org.htmlunit:htmlunit:4.9.0" + + /** Debugging **/ // Memory leak detection debugImplementation "com.squareup.leakcanary:leakcanary-object-watcher-android:${leakCanaryVersion}" diff --git a/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenGenerator.kt b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenGenerator.kt index 6446ecc72fd..3ec4790c5a0 100644 --- a/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenGenerator.kt +++ b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenGenerator.kt @@ -1,26 +1,41 @@ package org.schabi.newpipe.util.potoken import android.content.Context +import android.util.Log +import androidx.annotation.CallSuper +import io.reactivex.rxjava3.android.schedulers.AndroidSchedulers import io.reactivex.rxjava3.core.Single +import io.reactivex.rxjava3.core.SingleEmitter +import io.reactivex.rxjava3.disposables.CompositeDisposable +import io.reactivex.rxjava3.schedulers.Schedulers +import org.schabi.newpipe.BuildConfig +import org.schabi.newpipe.DownloaderImpl import java.io.Closeable +import java.time.Instant /** * This interface was created to allow for multiple methods to generate poTokens in the future (e.g. * via WebView and via a local DOM implementation) */ -interface PoTokenGenerator : Closeable { +abstract class PoTokenGenerator( + protected val generatorEmitter: SingleEmitter, +) : Closeable { + + protected val disposables = CompositeDisposable() // used only during initialization + private lateinit var expirationInstant: Instant + /** * Generates a poToken for the provided identifier, using the `integrityToken` and * `webPoSignalOutput` previously obtained in the initialization of [PoTokenWebView]. Can be * called multiple times. */ - fun generatePoToken(identifier: String): Single + abstract fun generatePoToken(identifier: String): Single /** * @return whether the `integrityToken` is expired, in which case all tokens generated by * [generatePoToken] will be invalid */ - fun isExpired(): Boolean + fun isExpired(): Boolean = Instant.now().isAfter(expirationInstant) interface Factory { /** @@ -32,4 +47,126 @@ interface PoTokenGenerator : Closeable { */ fun newPoTokenGenerator(context: Context): Single } + + //region Load HTML + protected fun loadPoTokenHtml(context: Context, handleHtml: (html: String) -> Unit) { + disposables.add( + Single.fromCallable { + val html = context.assets.open("po_token.html").bufferedReader() + .use { it.readText() } + return@fromCallable html + } + .subscribeOn(Schedulers.io()) + .observeOn(AndroidSchedulers.mainThread()) + .subscribe(handleHtml, this::onInitializationErrorCloseAndCancel) + ) + } + //endregion + + //region Network requests + /** + * Makes a POST request to [url] with the given [data] by setting the correct headers. Calls + * [onInitializationErrorCloseAndCancel] in case of any network errors and also if the response + * does not have HTTP code 200, therefore this is supposed to be used only during + * initialization. Calls [handleResponseBody] with the response body if the response is + * successful. The request is performed in the background and a disposable is added to + * [disposables]. + */ + private fun makeBotguardServiceRequest( + url: String, + data: String, + handleResponseBody: (responseBody: String) -> Unit, + ) { + disposables.add( + Single.fromCallable { + return@fromCallable DownloaderImpl.getInstance().post( + url, + mapOf( + // replace the downloader user agent + "User-Agent" to listOf(USER_AGENT), + "Accept" to listOf("application/json"), + "Content-Type" to listOf("application/json+protobuf"), + "x-goog-api-key" to listOf(GOOGLE_API_KEY), + "x-user-agent" to listOf("grpc-web-javascript/0.1"), + ), + data.toByteArray() + ) + } + .subscribeOn(Schedulers.io()) + .observeOn(AndroidSchedulers.mainThread()) + .subscribe( + { response -> + val httpCode = response.responseCode() + if (httpCode != 200) { + onInitializationErrorCloseAndCancel( + PoTokenException("Invalid response code: $httpCode") + ) + return@subscribe + } + val responseBody = response.responseBody() + handleResponseBody(responseBody) + }, + this::onInitializationErrorCloseAndCancel + ) + ) + } + + protected fun makeBotguardCreateRequest( + handleResponseBody: (responseBody: String) -> Unit + ) { + makeBotguardServiceRequest( + "https://www.youtube.com/api/jnn/v1/Create", + "[ \"$REQUEST_KEY\" ]", + handleResponseBody + ) + } + + protected fun makeBotguardGenerateITRequest( + botguardResponse: String, + handleIntegrityToken: (integrityToken: String) -> Unit + ) { + makeBotguardServiceRequest( + "https://www.youtube.com/api/jnn/v1/GenerateIT", + "[ \"$REQUEST_KEY\", \"$botguardResponse\" ]", + ) { responseBody -> + if (BuildConfig.DEBUG) { + Log.d(TAG, "GenerateIT response: $responseBody") + } + + val (integrityToken, expirationTimeInSeconds) = parseIntegrityTokenData(responseBody) + + // leave 10 minutes of margin just to be sure + expirationInstant = Instant.now().plusSeconds(expirationTimeInSeconds - 600) + + handleIntegrityToken(integrityToken) + } + } + //endregion + + //region Close + /** + * Handles any error happening during initialization, releasing resources as [close] would do + * and sending the error to [generatorEmitter] after releasing the resources has completed. + */ + protected abstract fun onInitializationErrorCloseAndCancel(error: Throwable) + + /** + * Disposes [disposables], disposing any network/io request. Must be overridden to also release + * the resources of the used web browser implementation (if any); remember `super.close()`! + */ + @CallSuper + override fun close() { + disposables.dispose() + } + //endregion + + companion object { + val TAG = PoTokenGenerator::class.simpleName + + // Public API key used by BotGuard, which has been got by looking at BotGuard requests + const val GOOGLE_API_KEY = "AIzaSyDyT5W0Jh49F30Pqqtyfdf7pDLFKLJoAnw" // NOSONAR + const val REQUEST_KEY = "O43z0dpjhgX20SCx4KAo" + const val USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + + "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.3" + } } diff --git a/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenHtmlUnit.kt b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenHtmlUnit.kt new file mode 100644 index 00000000000..912e6f326be --- /dev/null +++ b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenHtmlUnit.kt @@ -0,0 +1,173 @@ +package org.schabi.newpipe.util.potoken + +import android.content.Context +import android.util.Log +import androidx.annotation.MainThread +import io.reactivex.rxjava3.core.Single +import io.reactivex.rxjava3.core.SingleEmitter +import org.htmlunit.BrowserVersion +import org.htmlunit.BrowserVersion.BrowserVersionBuilder +import org.htmlunit.StringWebResponse +import org.htmlunit.WebClient +import org.htmlunit.html.HtmlPage +import org.schabi.newpipe.BuildConfig +import java.net.URL + +class PoTokenHtmlUnit private constructor( + // to be used exactly once only during initialization! + generatorEmitter: SingleEmitter, +) : PoTokenGenerator(generatorEmitter) { + private val webClient: WebClient + private lateinit var webPage: HtmlPage + + //region Initialization + init { + val browserVersion = BrowserVersionBuilder(BrowserVersion.CHROME) + .setUserAgent(USER_AGENT) + .build() + webClient = WebClient(browserVersion) + + // We are using alert() calls to send data from JavaScript to Java from async contexts, + // since HtmlUnit does not provide any other way for JavaScript to communicate with Java + // asynchronously. Note that this is only needed during initialization. + webClient.setAlertHandler { _, message -> + val argv = message.split(' ') + when (argv[0]) { + "onJsInitializationError" -> { onJsInitializationError(argv[1]) } + "onRunBotguardResult" -> { onRunBotguardResult(argv[1]) } + } + } + } + + /** + * Must be called right after instantiating [PoTokenHtmlUnit] to perform the actual + * initialization. This will asynchronously go through all the steps needed to load BotGuard, + * run it, and obtain an `integrityToken`. + */ + private fun loadHtmlAndObtainBotguard(context: Context) { + loadPoTokenHtml(context) { html -> + webPage = webClient.loadWebResponseInto( + StringWebResponse(html.replace("this", "window"), URL("https://www.youtube.com")), + webClient.currentWindow.topWindow, + ) as HtmlPage + webClient.javaScriptEngine + .registerWindowAndMaybeStartEventLoop(webClient.currentWindow.topWindow) + downloadAndRunBotguard() + } + } + + /** + * Called during initialization after the HtmlUnit [webPage] content has been loaded. + */ + private fun downloadAndRunBotguard() { + if (BuildConfig.DEBUG) { + Log.d(TAG, "downloadAndRunBotguard() called") + } + + makeBotguardCreateRequest { responseBody -> + val parsedChallengeData = parseChallengeData(responseBody) + webPage.executeJavaScript( + """try { + data = $parsedChallengeData + runBotGuard(data).then(function (result) { + window.webPoSignalOutput = result.webPoSignalOutput + alert("onRunBotguardResult " + result.botguardResponse) + }, function (error) { + alert("onJsInitializationError " + error + "\n" + error.stack) + }) + } catch (error) { + alert("onJsInitializationError " + error + "\n" + error.stack) + }""" + ) + } + } + + /** + * Called during initialization by the JavaScript snippet from [downloadAndRunBotguard]. + * Note: the communication from JavaScript to Java relies on `alert()` calls, handled by + * [WebClient.setAlertHandler]. + */ + private fun onJsInitializationError(error: String) { + Log.e(TAG, "Initialization error from JavaScript: $error") + onInitializationErrorCloseAndCancel(PoTokenException(error)) + } + + /** + * Called during initialization by the JavaScript snippet from [downloadAndRunBotguard] after + * obtaining the BotGuard execution output [botguardResponse]. + * Note: the communication from JavaScript to Java relies on `alert()` calls, handled by + * [WebClient.setAlertHandler]. + */ + private fun onRunBotguardResult(botguardResponse: String) { + if (BuildConfig.DEBUG) { + Log.d(TAG, "botguardResponse: $botguardResponse") + } + + makeBotguardGenerateITRequest(botguardResponse) { integrityToken -> + webPage.executeJavaScript( + "window.integrityToken = $integrityToken" + ) + + if (BuildConfig.DEBUG) { + Log.d(TAG, "initialization finished") + } + generatorEmitter.onSuccess(this) + } + } + //endregion + + //region Obtaining poTokens + @MainThread + override fun generatePoToken(identifier: String): Single = + Single.fromCallable { + val u8Identifier = stringToU8(identifier) + val result = webPage.executeJavaScript( + """result = "" + try { + identifier = "$identifier" + u8Identifier = $u8Identifier + poTokenU8 = obtainPoToken(window.webPoSignalOutput, window.integrityToken, u8Identifier) + poTokenU8String = "" + for (i = 0; i < poTokenU8.length; i++) { + if (i != 0) poTokenU8String += "," + poTokenU8String += poTokenU8[i] + } + result = poTokenU8String + } catch (error) { + result = "error " + error + "\n" + error.stack + "\n" + } + result""", + ).javaScriptResult.toString() + + if (result.startsWith("error ")) { + throw PoTokenException(result.substring(6)) + } else { + return@fromCallable u8ToBase64(result) + } + } + //endregion + + //region Close + override fun onInitializationErrorCloseAndCancel(error: Throwable) { + close() + generatorEmitter.onError(error) + } + + override fun close() { + super.close() + webClient.close() + } + //endregion + + companion object : Factory { + private val TAG = PoTokenHtmlUnit::class.simpleName + + @MainThread + override fun newPoTokenGenerator(context: Context): Single = + Single.create { emitter -> + val potWv = PoTokenHtmlUnit(emitter) + potWv.loadHtmlAndObtainBotguard(context) + emitter.setDisposable(potWv.disposables) + } + } +} diff --git a/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenProviderImpl.kt b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenProviderImpl.kt index 5383a613a86..ea0b7013b13 100644 --- a/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenProviderImpl.kt +++ b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenProviderImpl.kt @@ -76,7 +76,7 @@ object PoTokenProviderImpl : PoTokenProvider { webPoTokenGenerator?.let { Handler(Looper.getMainLooper()).post { it.close() } } // create a new webPoTokenGenerator - webPoTokenGenerator = PoTokenWebView + webPoTokenGenerator = PoTokenHtmlUnit .newPoTokenGenerator(App.getApp()).blockingGet() // The streaming poToken needs to be generated exactly once before generating diff --git a/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt index 9b4b500f09d..14a75ad4535 100644 --- a/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt +++ b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt @@ -11,24 +11,17 @@ import android.webkit.WebView import androidx.annotation.MainThread import androidx.webkit.WebSettingsCompat import androidx.webkit.WebViewFeature -import io.reactivex.rxjava3.android.schedulers.AndroidSchedulers import io.reactivex.rxjava3.core.Single import io.reactivex.rxjava3.core.SingleEmitter -import io.reactivex.rxjava3.disposables.CompositeDisposable -import io.reactivex.rxjava3.schedulers.Schedulers import org.schabi.newpipe.BuildConfig -import org.schabi.newpipe.DownloaderImpl -import java.time.Instant class PoTokenWebView private constructor( context: Context, // to be used exactly once only during initialization! - private val generatorEmitter: SingleEmitter, -) : PoTokenGenerator { + generatorEmitter: SingleEmitter, +) : PoTokenGenerator(generatorEmitter) { private val webView = WebView(context) - private val disposables = CompositeDisposable() // used only during initialization private val poTokenEmitters = mutableListOf>>() - private lateinit var expirationInstant: Instant //region Initialization init { @@ -74,31 +67,19 @@ class PoTokenWebView private constructor( Log.d(TAG, "loadHtmlAndObtainBotguard() called") } - disposables.add( - Single.fromCallable { - val html = context.assets.open("po_token.html").bufferedReader() - .use { it.readText() } - return@fromCallable html - } - .subscribeOn(Schedulers.io()) - .observeOn(AndroidSchedulers.mainThread()) - .subscribe( - { html -> - webView.loadDataWithBaseURL( - "https://www.youtube.com", - html.replaceFirst( - "", - // calls downloadAndRunBotguard() when the page has finished loading - "\n$JS_INTERFACE.downloadAndRunBotguard()" - ), - "text/html", - "utf-8", - null, - ) - }, - this::onInitializationErrorCloseAndCancel - ) - ) + loadPoTokenHtml(context) { html -> + webView.loadDataWithBaseURL( + "https://www.youtube.com", + html.replaceFirst( + "", + // calls downloadAndRunBotguard() when the page has finished loading + "\n$JS_INTERFACE.downloadAndRunBotguard()" + ), + "text/html", + "utf-8", + null, + ) + } } /** @@ -111,10 +92,7 @@ class PoTokenWebView private constructor( Log.d(TAG, "downloadAndRunBotguard() called") } - makeBotguardServiceRequest( - "https://www.youtube.com/api/jnn/v1/Create", - "[ \"$REQUEST_KEY\" ]", - ) { responseBody -> + makeBotguardCreateRequest { responseBody -> val parsedChallengeData = parseChallengeData(responseBody) webView.evaluateJavascript( """try { @@ -134,8 +112,7 @@ class PoTokenWebView private constructor( } /** - * Called during initialization by the JavaScript snippets from either - * [downloadAndRunBotguard] or [onRunBotguardResult]. + * Called during initialization by the JavaScript snippet from [downloadAndRunBotguard]. */ @JavascriptInterface fun onJsInitializationError(error: String) { @@ -154,23 +131,13 @@ class PoTokenWebView private constructor( if (BuildConfig.DEBUG) { Log.d(TAG, "botguardResponse: $botguardResponse") } - makeBotguardServiceRequest( - "https://www.youtube.com/api/jnn/v1/GenerateIT", - "[ \"$REQUEST_KEY\", \"$botguardResponse\" ]", - ) { responseBody -> - if (BuildConfig.DEBUG) { - Log.d(TAG, "GenerateIT response: $responseBody") - } - val (integrityToken, expirationTimeInSeconds) = parseIntegrityTokenData(responseBody) - - // leave 10 minutes of margin just to be sure - expirationInstant = Instant.now().plusSeconds(expirationTimeInSeconds - 600) + makeBotguardGenerateITRequest(botguardResponse) { integrityToken -> webView.evaluateJavascript( "this.integrityToken = $integrityToken" ) { if (BuildConfig.DEBUG) { - Log.d(TAG, "initialization finished, expiration=${expirationTimeInSeconds}s") + Log.d(TAG, "initialization finished") } generatorEmitter.onSuccess(this) } @@ -238,10 +205,6 @@ class PoTokenWebView private constructor( } popPoTokenEmitter(identifier)?.onSuccess(poToken) } - - override fun isExpired(): Boolean { - return Instant.now().isAfter(expirationInstant) - } //endregion //region Handling multiple emitters @@ -282,72 +245,12 @@ class PoTokenWebView private constructor( } //endregion - //region Utils - /** - * Makes a POST request to [url] with the given [data] by setting the correct headers. Calls - * [onInitializationErrorCloseAndCancel] in case of any network errors and also if the response - * does not have HTTP code 200, therefore this is supposed to be used only during - * initialization. Calls [handleResponseBody] with the response body if the response is - * successful. The request is performed in the background and a disposable is added to - * [disposables]. - */ - private fun makeBotguardServiceRequest( - url: String, - data: String, - handleResponseBody: (String) -> Unit, - ) { - disposables.add( - Single.fromCallable { - return@fromCallable DownloaderImpl.getInstance().post( - url, - mapOf( - // replace the downloader user agent - "User-Agent" to listOf(USER_AGENT), - "Accept" to listOf("application/json"), - "Content-Type" to listOf("application/json+protobuf"), - "x-goog-api-key" to listOf(GOOGLE_API_KEY), - "x-user-agent" to listOf("grpc-web-javascript/0.1"), - ), - data.toByteArray() - ) - } - .subscribeOn(Schedulers.io()) - .observeOn(AndroidSchedulers.mainThread()) - .subscribe( - { response -> - val httpCode = response.responseCode() - if (httpCode != 200) { - onInitializationErrorCloseAndCancel( - PoTokenException("Invalid response code: $httpCode") - ) - return@subscribe - } - val responseBody = response.responseBody() - handleResponseBody(responseBody) - }, - this::onInitializationErrorCloseAndCancel - ) - ) - } - - /** - * Handles any error happening during initialization, releasing resources and sending the error - * to [generatorEmitter]. - */ - private fun onInitializationErrorCloseAndCancel(error: Throwable) { - runOnMainThread(generatorEmitter) { - close() - generatorEmitter.onError(error) - } - } - + //region Close /** - * Releases all [webView] and [disposables] resources. + * Even clearing the [webView] needs to run on the main thread. */ @MainThread - override fun close() { - disposables.dispose() - + fun clearWebView() { webView.clearHistory() // clears RAM cache and disk cache (globally for all WebViews) webView.clearCache(true) @@ -359,15 +262,32 @@ class PoTokenWebView private constructor( webView.removeAllViews() webView.destroy() } + + override fun onInitializationErrorCloseAndCancel(error: Throwable) { + super.close() + runOnMainThread(generatorEmitter) { + try { + clearWebView() + } catch (t: Throwable) { + // ignore errors while clearing the WebView + Log.e(TAG, "Error while clearing webView", t) + } + + // only emit the error after clearing the webView has finished + generatorEmitter.onError(error) + } + } + + override fun close() { + super.close() + runOnMainThread(null) { + clearWebView() + } + } //endregion - companion object : PoTokenGenerator.Factory { + companion object : Factory { private val TAG = PoTokenWebView::class.simpleName - // Public API key used by BotGuard, which has been got by looking at BotGuard requests - private const val GOOGLE_API_KEY = "AIzaSyDyT5W0Jh49F30Pqqtyfdf7pDLFKLJoAnw" // NOSONAR - private const val REQUEST_KEY = "O43z0dpjhgX20SCx4KAo" - private const val USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + - "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.3" private const val JS_INTERFACE = "PoTokenWebView" override fun newPoTokenGenerator(context: Context): Single = @@ -384,11 +304,11 @@ class PoTokenWebView private constructor( * if the `post` fails emits an error on [emitterIfPostFails]. */ private fun runOnMainThread( - emitterIfPostFails: SingleEmitter, + emitterIfPostFails: SingleEmitter?, runnable: Runnable, ) { if (!Handler(Looper.getMainLooper()).post(runnable)) { - emitterIfPostFails.onError(PoTokenException("Could not run on main thread")) + emitterIfPostFails?.onError(PoTokenException("Could not run on main thread")) } } }