Skip to content

Commit db1955d

Browse files
BnyroBnyro
authored
Merge pull request libre-tube#7112 from Briiqn/master
feat: optimize extraction performance & simplify code
2 parents b27d518 + 24a0635 commit db1955d

3 files changed

Lines changed: 118 additions & 106 deletions

File tree

‎app/src/main/java/com/github/libretube/api/StreamsExtractor.kt‎

Lines changed: 51 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -13,17 +13,22 @@ import com.github.libretube.api.obj.Subtitle
1313
import com.github.libretube.extensions.toID
1414
import com.github.libretube.helpers.PlayerHelper
1515
import com.github.libretube.ui.dialogs.ShareDialog.Companion.YOUTUBE_FRONTEND_URL
16+
import com.github.libretube.util.deArrow
17+
import kotlinx.coroutines.Dispatchers
18+
import kotlinx.coroutines.async
19+
import kotlinx.coroutines.withContext
1620
import kotlinx.datetime.toKotlinInstant
21+
import org.schabi.newpipe.extractor.stream.AudioStream
1722
import org.schabi.newpipe.extractor.stream.StreamInfo
1823
import org.schabi.newpipe.extractor.stream.StreamInfoItem
1924
import org.schabi.newpipe.extractor.stream.VideoStream
2025
import retrofit2.HttpException
2126
import java.io.IOException
2227

23-
fun VideoStream.toPipedStream(): PipedStream = PipedStream(
28+
fun VideoStream.toPipedStream() = PipedStream(
2429
url = content,
2530
codec = codec,
26-
format = format.toString(),
31+
format = format?.toString(),
2732
height = height,
2833
width = width,
2934
quality = getResolution(),
@@ -37,14 +42,34 @@ fun VideoStream.toPipedStream(): PipedStream = PipedStream(
3742
contentLength = itagItem?.contentLength ?: 0L
3843
)
3944

45+
fun AudioStream.toPipedStream() = PipedStream(
46+
url = content,
47+
format = format?.toString(),
48+
quality = "$averageBitrate bits",
49+
bitrate = bitrate,
50+
mimeType = format?.mimeType,
51+
initStart = initStart,
52+
initEnd = initEnd,
53+
indexStart = indexStart,
54+
indexEnd = indexEnd,
55+
contentLength = itagItem?.contentLength ?: 0L,
56+
codec = codec,
57+
audioTrackId = audioTrackId,
58+
audioTrackName = audioTrackName,
59+
audioTrackLocale = audioLocale?.toLanguageTag(),
60+
audioTrackType = audioTrackType?.name,
61+
videoOnly = false
62+
)
63+
4064
fun StreamInfoItem.toStreamItem(
4165
uploaderAvatarUrl: String? = null
42-
): StreamItem = StreamItem(
66+
) = StreamItem(
4367
type = StreamItem.TYPE_STREAM,
4468
url = url.toID(),
4569
title = name,
4670
uploaded = uploadDate?.offsetDateTime()?.toEpochSecond()?.times(1000) ?: -1,
47-
uploadedDate = textualUploadDate ?: uploadDate?.offsetDateTime()?.toLocalDateTime()?.toLocalDate()
71+
uploadedDate = textualUploadDate ?: uploadDate?.offsetDateTime()?.toLocalDateTime()
72+
?.toLocalDate()
4873
?.toString(),
4974
uploaderName = uploaderName,
5075
uploaderUrl = uploaderUrl.toID(),
@@ -58,13 +83,22 @@ fun StreamInfoItem.toStreamItem(
5883
)
5984

6085
object StreamsExtractor {
61-
suspend fun extractStreams(videoId: String): Streams {
86+
suspend fun extractStreams(videoId: String): Streams = withContext(Dispatchers.IO) {
6287
if (!PlayerHelper.disablePipedProxy || !PlayerHelper.localStreamExtraction) {
63-
return RetrofitInstance.api.getStreams(videoId)
88+
return@withContext RetrofitInstance.api.getStreams(videoId).deArrow(videoId)
6489
}
6590

66-
val resp = StreamInfo.getInfo("${YOUTUBE_FRONTEND_URL}/watch?v=$videoId")
67-
return Streams(
91+
val respAsync = async {
92+
StreamInfo.getInfo("$YOUTUBE_FRONTEND_URL/watch?v=$videoId")
93+
}
94+
val dislikesAsync = async {
95+
if (PlayerHelper.localRYD) runCatching {
96+
RetrofitInstance.externalApi.getVotes(videoId).dislikes
97+
}.getOrElse { -1 } else -1
98+
}
99+
val (resp, dislikes) = Pair(respAsync.await(), dislikesAsync.await())
100+
101+
Streams(
68102
title = resp.name,
69103
description = resp.description.content,
70104
uploader = resp.uploaderName,
@@ -75,9 +109,7 @@ object StreamsExtractor {
75109
category = resp.category,
76110
views = resp.viewCount,
77111
likes = resp.likeCount,
78-
dislikes = if (PlayerHelper.localRYD) runCatching {
79-
RetrofitInstance.externalApi.getVotes(videoId).dislikes
80-
}.getOrElse { -1 } else -1,
112+
dislikes = dislikes,
81113
license = resp.licence,
82114
hls = resp.hlsUrl,
83115
dash = resp.dashMpdUrl,
@@ -95,39 +127,19 @@ object StreamsExtractor {
95127
uploadTimestamp = resp.uploadDate.offsetDateTime().toInstant().toKotlinInstant(),
96128
uploaded = resp.uploadDate.offsetDateTime().toEpochSecond() * 1000,
97129
thumbnailUrl = resp.thumbnails.maxBy { it.height }.url,
98-
relatedStreams = resp.relatedItems.filterIsInstance<StreamInfoItem>().map(StreamInfoItem::toStreamItem),
130+
relatedStreams = resp.relatedItems
131+
.filterIsInstance<StreamInfoItem>()
132+
.map { item -> item.toStreamItem() },
99133
chapters = resp.streamSegments.map {
100134
ChapterSegment(
101135
title = it.title,
102136
image = it.previewUrl.orEmpty(),
103137
start = it.startTimeSeconds.toLong()
104138
)
105139
},
106-
audioStreams = resp.audioStreams.map {
107-
PipedStream(
108-
url = it.content,
109-
format = it.format?.toString(),
110-
quality = "${it.averageBitrate} bits",
111-
bitrate = it.bitrate,
112-
mimeType = it.format?.mimeType,
113-
initStart = it.initStart,
114-
initEnd = it.initEnd,
115-
indexStart = it.indexStart,
116-
indexEnd = it.indexEnd,
117-
contentLength = it.itagItem?.contentLength ?: 0L,
118-
codec = it.codec,
119-
audioTrackId = it.audioTrackId,
120-
audioTrackName = it.audioTrackName,
121-
audioTrackLocale = it.audioLocale?.toLanguageTag(),
122-
audioTrackType = it.audioTrackType?.name,
123-
videoOnly = false
124-
)
125-
},
126-
videoStreams = resp.videoOnlyStreams.map {
127-
it.toPipedStream().copy(videoOnly = true)
128-
} + resp.videoStreams.map {
129-
it.toPipedStream().copy(videoOnly = false)
130-
},
140+
audioStreams = resp.audioStreams.map { it.toPipedStream() },
141+
videoStreams = resp.videoOnlyStreams.map { it.toPipedStream().copy(videoOnly = true) } +
142+
resp.videoStreams.map { it.toPipedStream().copy(videoOnly = false) },
131143
previewFrames = resp.previewFrames.map {
132144
PreviewFrames(
133145
it.urls,
@@ -148,7 +160,7 @@ object StreamsExtractor {
148160
it.isAutoGenerated
149161
)
150162
}
151-
)
163+
).deArrow(videoId)
152164
}
153165

154166
fun getExtractorErrorMessageString(context: Context, exception: Exception): String {
@@ -157,6 +169,7 @@ object StreamsExtractor {
157169
is HttpException -> exception.response()?.errorBody()?.string()?.runCatching {
158170
JsonHelper.json.decodeFromString<Message>(this).message
159171
}?.getOrNull() ?: context.getString(R.string.server_error)
172+
160173
else -> exception.localizedMessage.orEmpty()
161174
}
162175
}

‎app/src/main/java/com/github/libretube/util/DeArrowUtil.kt‎

Lines changed: 37 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -13,20 +13,22 @@ import java.util.TreeSet
1313

1414
object DeArrowUtil {
1515
private fun extractTitleAndThumbnail(content: DeArrowContent): Pair<String?, String?> {
16-
val newTitle = content.titles.firstOrNull { it.votes >= 0 || it.locked }?.title
17-
val newThumbnail = content.thumbnails.firstOrNull {
16+
val title = content.titles.firstOrNull { it.votes >= 0 || it.locked }?.title
17+
val thumbnail = content.thumbnails.firstOrNull {
1818
it.thumbnail != null && !it.original && (it.votes >= 0 || it.locked)
1919
}?.thumbnail
20-
return newTitle to newThumbnail
20+
21+
return title to thumbnail
2122
}
2223

24+
2325
private suspend fun fetchDeArrowContent(videoIds: List<String>): Map<String, DeArrowContent>? {
2426
val videoIdsString = videoIds.mapTo(TreeSet()) { it }.joinToString(",")
2527

2628
return try {
2729
RetrofitInstance.api.getDeArrowContent(videoIdsString)
2830
} catch (e: Exception) {
29-
Log.e(this::class.java.name, e.toString())
31+
Log.e(this::class.java.name, "Failed to fetch DeArrow content: ${e.message}")
3032
null
3133
}
3234
}
@@ -37,19 +39,19 @@ object DeArrowUtil {
3739
suspend fun deArrowStreams(streams: Streams, vidId: String): Streams {
3840
if (!PreferenceHelper.getBoolean(PreferenceKeys.DEARROW, false)) return streams
3941

40-
val videoIds = listOf(vidId) + streams.relatedStreams.map { it.url!!.toID() }
42+
val videoIds = listOf(vidId) + streams.relatedStreams.mapNotNull { it.url?.toID() }
4143
val response = fetchDeArrowContent(videoIds) ?: return streams
4244

43-
for ((videoId, data) in response.entries) {
45+
response[vidId]?.let { data ->
4446
val (newTitle, newThumbnail) = extractTitleAndThumbnail(data)
47+
if (newTitle != null) streams.title = newTitle
48+
if (newThumbnail != null) streams.thumbnailUrl = newThumbnail
49+
}
4550

46-
if (videoId == vidId) {
47-
if (newTitle != null) streams.title = newTitle
48-
if (newThumbnail != null) streams.thumbnailUrl = newThumbnail
49-
} else {
50-
val streamItem = streams.relatedStreams
51-
.firstOrNull { it.url?.toID() == videoId } ?: continue
52-
51+
streams.relatedStreams.forEach { streamItem ->
52+
val videoId = streamItem.url?.toID() ?: return@forEach
53+
response[videoId]?.let { data ->
54+
val (newTitle, newThumbnail) = extractTitleAndThumbnail(data)
5355
if (newTitle != null) streamItem.title = newTitle
5456
if (newThumbnail != null) streamItem.thumbnail = newThumbnail
5557
}
@@ -63,54 +65,53 @@ object DeArrowUtil {
6365
*/
6466
suspend fun deArrowStreamItems(streamItems: List<StreamItem>): List<StreamItem> {
6567
if (!PreferenceHelper.getBoolean(PreferenceKeys.DEARROW, false)) return streamItems
68+
if (streamItems.isEmpty()) return streamItems
6669

67-
val response = fetchDeArrowContent(streamItems.map{ it.url!!.toID() }) ?: return streamItems
70+
val videoIds = streamItems.mapNotNull { it.url?.toID() }
71+
val response = fetchDeArrowContent(videoIds) ?: return streamItems
6872

69-
for ((videoId, data) in response.entries) {
70-
val (newTitle, newThumbnail) = extractTitleAndThumbnail(data)
71-
val streamItem = streamItems.firstOrNull { it.url?.toID() == videoId } ?: continue
72-
73-
if (newTitle != null) streamItem.title = newTitle
74-
if (newThumbnail != null) streamItem.thumbnail = newThumbnail
73+
streamItems.forEach { streamItem ->
74+
val videoId = streamItem.url?.toID() ?: return@forEach
75+
response[videoId]?.let { data ->
76+
val (newTitle, newThumbnail) = extractTitleAndThumbnail(data)
77+
if (newTitle != null) streamItem.title = newTitle
78+
if (newThumbnail != null) streamItem.thumbnail = newThumbnail
79+
}
7580
}
81+
7682
return streamItems
7783
}
7884

7985
/**
80-
* Apply the new titles and thumbnails generated by DeArrow to the stream items
86+
* Apply the new titles and thumbnails generated by DeArrow to the content items
8187
*/
8288
suspend fun deArrowContentItems(contentItems: List<ContentItem>): List<ContentItem> {
8389
if (!PreferenceHelper.getBoolean(PreferenceKeys.DEARROW, false)) return contentItems
8490

85-
val videoIds = contentItems.filter { it.type == "stream" }
91+
val videoIds = contentItems
92+
.filter { it.type == "stream" }
8693
.map { it.url.toID() }
8794

8895
if (videoIds.isEmpty()) return contentItems
8996

9097
val response = fetchDeArrowContent(videoIds) ?: return contentItems
9198

92-
for ((videoId, data) in response.entries) {
93-
val (newTitle, newThumbnail) = extractTitleAndThumbnail(data)
94-
val contentItem = contentItems.firstOrNull { it.url.toID() == videoId } ?: continue
95-
96-
if (newTitle != null) { contentItem.title = newTitle }
97-
if (newThumbnail != null) { contentItem.thumbnail = newThumbnail }
99+
contentItems.forEach { contentItem ->
100+
val videoId = contentItem.url.toID()
101+
response[videoId]?.let { data ->
102+
val (newTitle, newThumbnail) = extractTitleAndThumbnail(data)
103+
if (newTitle != null) contentItem.title = newTitle
104+
if (newThumbnail != null) contentItem.thumbnail = newThumbnail
105+
}
98106
}
107+
99108
return contentItems
100109
}
101110
}
102111

103-
/**
104-
* If enabled in the preferences, this overrides the video's thumbnail and title with the one
105-
* provided by the DeArrow project
106-
*/
107112
@JvmName("deArrowStreamItems")
108113
suspend fun List<StreamItem>.deArrow() = DeArrowUtil.deArrowStreamItems(this)
109114

110-
/**
111-
* If enabled in the preferences, this overrides the video's thumbnail and title with the one
112-
* provided by the DeArrow project
113-
*/
114115
@JvmName("deArrowContentItems")
115116
suspend fun List<ContentItem>.deArrow() = DeArrowUtil.deArrowContentItems(this)
116117

‎app/src/main/java/com/github/libretube/util/NewPipeDownloaderImpl.kt‎

Lines changed: 30 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1,60 +1,58 @@
11
package com.github.libretube.util
22

3-
import java.io.IOException
4-
import java.util.concurrent.TimeUnit
5-
import okhttp3.MediaType.Companion.toMediaType
63
import okhttp3.OkHttpClient
74
import okhttp3.RequestBody.Companion.toRequestBody
85
import org.schabi.newpipe.extractor.downloader.Downloader
96
import org.schabi.newpipe.extractor.downloader.Request
107
import org.schabi.newpipe.extractor.downloader.Response
118
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException
9+
import java.io.IOException
1210

1311
class NewPipeDownloaderImpl : Downloader() {
14-
private val client: OkHttpClient = OkHttpClient.Builder()
15-
.readTimeout(READ_TIMEOUT_SECONDS, TimeUnit.SECONDS)
12+
private val client = OkHttpClient.Builder()
1613
.build()
1714

1815
@Throws(IOException::class, ReCaptchaException::class)
1916
override fun execute(request: Request): Response {
17+
val httpMethod = request.httpMethod()
2018
val url = request.url()
21-
22-
val requestBody = request.dataToSend()?.let {
23-
it.toRequestBody(APPLICATION_JSON, 0, it.size)
24-
}
19+
val headers = request.headers()
20+
val dataToSend = request.dataToSend()
2521

2622
val requestBuilder = okhttp3.Request.Builder()
27-
.method(request.httpMethod(), requestBody)
23+
.method(httpMethod, dataToSend?.toRequestBody())
2824
.url(url)
29-
.addHeader(USER_AGENT_HEADER_NAME, USER_AGENT)
25+
.addHeader("User-Agent", USER_AGENT)
3026

31-
for ((headerName, headerValueList) in request.headers()) {
32-
requestBuilder.removeHeader(headerName)
33-
for (headerValue in headerValueList) {
34-
requestBuilder.addHeader(headerName, headerValue)
27+
for ((headerKey, headerValues) in headers) {
28+
requestBuilder.removeHeader(headerKey)
29+
for (headerValue in headerValues) {
30+
requestBuilder.addHeader(headerKey, headerValue)
3531
}
3632
}
37-
3833
val response = client.newCall(requestBuilder.build()).execute()
39-
if (response.code == CAPTCHA_STATUS_CODE) {
40-
response.close()
41-
throw ReCaptchaException("reCaptcha Challenge requested", url)
42-
}
4334

44-
return Response(
45-
response.code,
46-
response.message,
47-
response.headers.toMultimap(),
48-
response.body?.string(),
49-
response.request.url.toString()
50-
)
35+
return when (response.code) {
36+
429 -> {
37+
response.close()
38+
throw ReCaptchaException("reCaptcha Challenge requested", url)
39+
}
40+
41+
else -> {
42+
val responseBodyToReturn = response.body?.string()
43+
Response(
44+
response.code,
45+
response.message,
46+
response.headers.toMultimap(),
47+
responseBodyToReturn,
48+
response.request.url.toString()
49+
)
50+
}
51+
}
5152
}
5253

5354
companion object {
54-
private const val USER_AGENT_HEADER_NAME = "User-Agent"
55-
private const val USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; rv:102.0) Gecko/20100101 Firefox/102.0"
56-
private const val CAPTCHA_STATUS_CODE = 429
57-
private val APPLICATION_JSON = "application/json".toMediaType()
58-
private const val READ_TIMEOUT_SECONDS = 30L
55+
private const val USER_AGENT =
56+
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:135.0) Gecko/20100101 Firefox/135.0"
5957
}
6058
}

0 commit comments

Comments
 (0)