[YouTube] PoToken implementation with HtmlUnit

app/build.gradle

@@ -18,7 +18,7 @@ android {
     defaultConfig {
         applicationId "org.schabi.newpipe"
         resValue "string", "app_name", "NewPipe"
-        minSdk 21
+        minSdk 26
         targetSdk 33
         if (System.properties.containsKey('versionCodeOverride')) {
             versionCode System.getProperty('versionCodeOverride') as Integer
@@ -108,7 +108,10 @@ android {
             // no idea how they ended up in the META-INF dir...
             excludes += ['META-INF/README.md', 'META-INF/CHANGES',
                          // 'COPYRIGHT' belongs to RxJava...
-                         'META-INF/COPYRIGHT']
+                         'META-INF/COPYRIGHT',
+                         // belongs
+                         'META-INF/DEPENDENCIES'
+                        ]
         }
     }
 }
@@ -296,6 +299,10 @@ dependencies {
     // Date and time formatting
     implementation "org.ocpsoft.prettytime:prettytime:5.0.8.Final"
 
+    // Web browser automation
+    implementation "org.htmlunit:htmlunit:4.9.0"
+
+
 /** Debugging **/
     // Memory leak detection
     debugImplementation "com.squareup.leakcanary:leakcanary-object-watcher-android:${leakCanaryVersion}"

app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenGenerator.kt

@@ -1,26 +1,41 @@
 package org.schabi.newpipe.util.potoken
 
 import android.content.Context
+import android.util.Log
+import androidx.annotation.CallSuper
+import io.reactivex.rxjava3.android.schedulers.AndroidSchedulers
 import io.reactivex.rxjava3.core.Single
+import io.reactivex.rxjava3.core.SingleEmitter
+import io.reactivex.rxjava3.disposables.CompositeDisposable
+import io.reactivex.rxjava3.schedulers.Schedulers
+import org.schabi.newpipe.BuildConfig
+import org.schabi.newpipe.DownloaderImpl
 import java.io.Closeable
+import java.time.Instant
 
 /**
  * This interface was created to allow for multiple methods to generate poTokens in the future (e.g.
  * via WebView and via a local DOM implementation)
  */
-interface PoTokenGenerator : Closeable {
+abstract class PoTokenGenerator(
+    protected val generatorEmitter: SingleEmitter<PoTokenGenerator>,
+) : Closeable {
+
+    protected val disposables = CompositeDisposable() // used only during initialization
+    private lateinit var expirationInstant: Instant
+
     /**
      * Generates a poToken for the provided identifier, using the `integrityToken` and
      * `webPoSignalOutput` previously obtained in the initialization of [PoTokenWebView]. Can be
      * called multiple times.
      */
-    fun generatePoToken(identifier: String): Single<String>
+    abstract fun generatePoToken(identifier: String): Single<String>
 
     /**
      * @return whether the `integrityToken` is expired, in which case all tokens generated by
      * [generatePoToken] will be invalid
      */
-    fun isExpired(): Boolean
+    fun isExpired(): Boolean = Instant.now().isAfter(expirationInstant)
 
     interface Factory {
         /**
@@ -32,4 +47,126 @@ interface PoTokenGenerator : Closeable {
          */
         fun newPoTokenGenerator(context: Context): Single<PoTokenGenerator>
     }
+
+    //region Load HTML
+    protected fun loadPoTokenHtml(context: Context, handleHtml: (html: String) -> Unit) {
+        disposables.add(
+            Single.fromCallable {
+                val html = context.assets.open("po_token.html").bufferedReader()
+                    .use { it.readText() }
+                return@fromCallable html
+            }
+                .subscribeOn(Schedulers.io())
+                .observeOn(AndroidSchedulers.mainThread())
+                .subscribe(handleHtml, this::onInitializationErrorCloseAndCancel)
+        )
+    }
+    //endregion
+
+    //region Network requests
+    /**
+     * Makes a POST request to [url] with the given [data] by setting the correct headers. Calls
+     * [onInitializationErrorCloseAndCancel] in case of any network errors and also if the response
+     * does not have HTTP code 200, therefore this is supposed to be used only during
+     * initialization. Calls [handleResponseBody] with the response body if the response is
+     * successful. The request is performed in the background and a disposable is added to
+     * [disposables].
+     */
+    private fun makeBotguardServiceRequest(
+        url: String,
+        data: String,
+        handleResponseBody: (responseBody: String) -> Unit,
+    ) {
+        disposables.add(
+            Single.fromCallable {
+                return@fromCallable DownloaderImpl.getInstance().post(
+                    url,
+                    mapOf(
+                        // replace the downloader user agent
+                        "User-Agent" to listOf(USER_AGENT),
+                        "Accept" to listOf("application/json"),
+                        "Content-Type" to listOf("application/json+protobuf"),
+                        "x-goog-api-key" to listOf(GOOGLE_API_KEY),
+                        "x-user-agent" to listOf("grpc-web-javascript/0.1"),
+                    ),
+                    data.toByteArray()
+                )
+            }
+                .subscribeOn(Schedulers.io())
+                .observeOn(AndroidSchedulers.mainThread())
+                .subscribe(
+                    { response ->
+                        val httpCode = response.responseCode()
+                        if (httpCode != 200) {
+                            onInitializationErrorCloseAndCancel(
+                                PoTokenException("Invalid response code: $httpCode")
+                            )
+                            return@subscribe
+                        }
+                        val responseBody = response.responseBody()
+                        handleResponseBody(responseBody)
+                    },
+                    this::onInitializationErrorCloseAndCancel
+                )
+        )
+    }
+
+    protected fun makeBotguardCreateRequest(
+        handleResponseBody: (responseBody: String) -> Unit
+    ) {
+        makeBotguardServiceRequest(
+            "https://www.youtube.com/api/jnn/v1/Create",
+            "[ \"$REQUEST_KEY\" ]",
+            handleResponseBody
+        )
+    }
+
+    protected fun makeBotguardGenerateITRequest(
+        botguardResponse: String,
+        handleIntegrityToken: (integrityToken: String) -> Unit
+    ) {
+        makeBotguardServiceRequest(
+            "https://www.youtube.com/api/jnn/v1/GenerateIT",
+            "[ \"$REQUEST_KEY\", \"$botguardResponse\" ]",
+        ) { responseBody ->
+            if (BuildConfig.DEBUG) {
+                Log.d(TAG, "GenerateIT response: $responseBody")
+            }
+
+            val (integrityToken, expirationTimeInSeconds) = parseIntegrityTokenData(responseBody)
+
+            // leave 10 minutes of margin just to be sure
+            expirationInstant = Instant.now().plusSeconds(expirationTimeInSeconds - 600)
+
+            handleIntegrityToken(integrityToken)
+        }
+    }
+    //endregion
+
+    //region Close
+    /**
+     * Handles any error happening during initialization, releasing resources as [close] would do
+     * and sending the error to [generatorEmitter] after releasing the resources has completed.
+     */
+    protected abstract fun onInitializationErrorCloseAndCancel(error: Throwable)
+
+    /**
+     * Disposes [disposables], disposing any network/io request. Must be overridden to also release
+     * the resources of the used web browser implementation (if any); remember `super.close()`!
+     */
+    @CallSuper
+    override fun close() {
+        disposables.dispose()
+    }
+    //endregion
+
+    companion object {
+        val TAG = PoTokenGenerator::class.simpleName
+
+        // Public API key used by BotGuard, which has been got by looking at BotGuard requests
+        const val GOOGLE_API_KEY = "AIzaSyDyT5W0Jh49F30Pqqtyfdf7pDLFKLJoAnw" // NOSONAR
+        const val REQUEST_KEY = "O43z0dpjhgX20SCx4KAo"
+        const val USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " +
+            "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.3"
+    }
 }

app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenHtmlUnit.kt

@@ -0,0 +1,173 @@
+package org.schabi.newpipe.util.potoken
+
+import android.content.Context
+import android.util.Log
+import androidx.annotation.MainThread
+import io.reactivex.rxjava3.core.Single
+import io.reactivex.rxjava3.core.SingleEmitter
+import org.htmlunit.BrowserVersion
+import org.htmlunit.BrowserVersion.BrowserVersionBuilder
+import org.htmlunit.StringWebResponse
+import org.htmlunit.WebClient
+import org.htmlunit.html.HtmlPage
+import org.schabi.newpipe.BuildConfig
+import java.net.URL
+
+class PoTokenHtmlUnit private constructor(
+    // to be used exactly once only during initialization!
+    generatorEmitter: SingleEmitter<PoTokenGenerator>,
+) : PoTokenGenerator(generatorEmitter) {
+    private val webClient: WebClient
+    private lateinit var webPage: HtmlPage
+
+    //region Initialization
+    init {
+        val browserVersion = BrowserVersionBuilder(BrowserVersion.CHROME)
+            .setUserAgent(USER_AGENT)
+            .build()
+        webClient = WebClient(browserVersion)
+
+        // We are using alert() calls to send data from JavaScript to Java from async contexts,
+        // since HtmlUnit does not provide any other way for JavaScript to communicate with Java
+        // asynchronously. Note that this is only needed during initialization.
+        webClient.setAlertHandler { _, message ->
+            val argv = message.split(' ')
+            when (argv[0]) {
+                "onJsInitializationError" -> { onJsInitializationError(argv[1]) }
+                "onRunBotguardResult" -> { onRunBotguardResult(argv[1]) }
+            }
+        }
+    }
+
+    /**
+     * Must be called right after instantiating [PoTokenHtmlUnit] to perform the actual
+     * initialization. This will asynchronously go through all the steps needed to load BotGuard,
+     * run it, and obtain an `integrityToken`.
+     */
+    private fun loadHtmlAndObtainBotguard(context: Context) {
+        loadPoTokenHtml(context) { html ->
+            webPage = webClient.loadWebResponseInto(
+                StringWebResponse(html.replace("this", "window"), URL("https://www.youtube.com")),
+                webClient.currentWindow.topWindow,
+            ) as HtmlPage
+            webClient.javaScriptEngine
+                .registerWindowAndMaybeStartEventLoop(webClient.currentWindow.topWindow)
+            downloadAndRunBotguard()
+        }
+    }
+
+    /**
+     * Called during initialization after the HtmlUnit [webPage] content has been loaded.
+     */
+    private fun downloadAndRunBotguard() {
+        if (BuildConfig.DEBUG) {
+            Log.d(TAG, "downloadAndRunBotguard() called")
+        }
+
+        makeBotguardCreateRequest { responseBody ->
+            val parsedChallengeData = parseChallengeData(responseBody)
+            webPage.executeJavaScript(
+                """try {
+                    data = $parsedChallengeData
+                    runBotGuard(data).then(function (result) {
+                        window.webPoSignalOutput = result.webPoSignalOutput
+                        alert("onRunBotguardResult " + result.botguardResponse)
+                    }, function (error) {
+                        alert("onJsInitializationError " + error + "\n" + error.stack)
+                    })
+                } catch (error) {
+                    alert("onJsInitializationError " + error + "\n" + error.stack)
+                }"""
+            )
+        }
+    }
+
+    /**
+     * Called during initialization by the JavaScript snippet from [downloadAndRunBotguard].
+     * Note: the communication from JavaScript to Java relies on `alert()` calls, handled by
+     * [WebClient.setAlertHandler].
+     */
+    private fun onJsInitializationError(error: String) {
+        Log.e(TAG, "Initialization error from JavaScript: $error")
+        onInitializationErrorCloseAndCancel(PoTokenException(error))
+    }
+
+    /**
+     * Called during initialization by the JavaScript snippet from [downloadAndRunBotguard] after
+     * obtaining the BotGuard execution output [botguardResponse].
+     * Note: the communication from JavaScript to Java relies on `alert()` calls, handled by
+     * [WebClient.setAlertHandler].
+     */
+    private fun onRunBotguardResult(botguardResponse: String) {
+        if (BuildConfig.DEBUG) {
+            Log.d(TAG, "botguardResponse: $botguardResponse")
+        }
+
+        makeBotguardGenerateITRequest(botguardResponse) { integrityToken ->
+            webPage.executeJavaScript(
+                "window.integrityToken = $integrityToken"
+            )
+
+            if (BuildConfig.DEBUG) {
+                Log.d(TAG, "initialization finished")
+            }
+            generatorEmitter.onSuccess(this)
+        }
+    }
+    //endregion
+
+    //region Obtaining poTokens
+    @MainThread
+    override fun generatePoToken(identifier: String): Single<String> =
+        Single.fromCallable {
+            val u8Identifier = stringToU8(identifier)
+            val result = webPage.executeJavaScript(
+                """result = ""
+                try {
+                    identifier = "$identifier"
+                    u8Identifier = $u8Identifier
+                    poTokenU8 = obtainPoToken(window.webPoSignalOutput, window.integrityToken, u8Identifier)
+                    poTokenU8String = ""
+                    for (i = 0; i < poTokenU8.length; i++) {
+                        if (i != 0) poTokenU8String += ","
+                        poTokenU8String += poTokenU8[i]
+                    }
+                    result = poTokenU8String
+                } catch (error) {
+                    result = "error " + error + "\n" + error.stack + "\n"
+                }
+                result""",
+            ).javaScriptResult.toString()
+
+            if (result.startsWith("error ")) {
+                throw PoTokenException(result.substring(6))
+            } else {
+                return@fromCallable u8ToBase64(result)
+            }
+        }
+    //endregion
+
+    //region Close
+    override fun onInitializationErrorCloseAndCancel(error: Throwable) {
+        close()
+        generatorEmitter.onError(error)
+    }
+
+    override fun close() {
+        super.close()
+        webClient.close()
+    }
+    //endregion
+
+    companion object : Factory {
+        private val TAG = PoTokenHtmlUnit::class.simpleName
+
+        @MainThread
+        override fun newPoTokenGenerator(context: Context): Single<PoTokenGenerator> =
+            Single.create { emitter ->
+                val potWv = PoTokenHtmlUnit(emitter)
+                potWv.loadHtmlAndObtainBotguard(context)
+                emitter.setDisposable(potWv.disposables)
+            }
+    }
+}

app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenProviderImpl.kt

@@ -76,7 +76,7 @@ object PoTokenProviderImpl : PoTokenProvider {
                     webPoTokenGenerator?.let { Handler(Looper.getMainLooper()).post { it.close() } }
 
                     // create a new webPoTokenGenerator
-                    webPoTokenGenerator = PoTokenWebView
+                    webPoTokenGenerator = PoTokenHtmlUnit
                         .newPoTokenGenerator(App.getApp()).blockingGet()
 
                     // The streaming poToken needs to be generated exactly once before generating