Skip to content

Commit 426a227

Browse files
StypoxStypox
authored
Merge pull request #1347 from litetex/fix-yt-channel-id-resolution
2 parents a5fcc7d + 6397b2e commit 426a227

11 files changed

Lines changed: 2621 additions & 25 deletions

File tree

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelHelper.java‎

Lines changed: 49 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,28 @@
11
package org.schabi.newpipe.extractor.services.youtube;
22

3+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.defaultAlertsCheck;
4+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
5+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
6+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.hasArtistOrVerifiedIconBadgeAttachment;
7+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
8+
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
9+
310
import com.grack.nanojson.JsonObject;
411
import com.grack.nanojson.JsonWriter;
12+
513
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
614
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
715
import org.schabi.newpipe.extractor.exceptions.ParsingException;
816
import org.schabi.newpipe.extractor.localization.ContentCountry;
917
import org.schabi.newpipe.extractor.localization.Localization;
1018

11-
import javax.annotation.Nonnull;
12-
import javax.annotation.Nullable;
1319
import java.io.IOException;
1420
import java.io.Serializable;
1521
import java.nio.charset.StandardCharsets;
1622
import java.util.Optional;
1723

18-
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.defaultAlertsCheck;
19-
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
20-
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
21-
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.hasArtistOrVerifiedIconBadgeAttachment;
22-
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
23-
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
24+
import javax.annotation.Nonnull;
25+
import javax.annotation.Nullable;
2426

2527
/**
2628
* Shared functions for extracting YouTube channel pages and tabs.
@@ -65,35 +67,59 @@ public static String resolveChannelId(@Nonnull final String idOrPath)
6567
// URL, then no information about the channel associated with this URL was found,
6668
// so the unresolved url will be returned.
6769
if (!channelId[0].equals("channel")) {
68-
final byte[] body = JsonWriter.string(
69-
prepareDesktopJsonBuilder(Localization.DEFAULT, ContentCountry.DEFAULT)
70-
.value("url", "https://www.youtube.com/" + idOrPath)
70+
String urlToResolve = "https://www.youtube.com/" + idOrPath;
71+
72+
JsonObject endpoint = new JsonObject();
73+
String webPageType = "";
74+
// Try to resolve YT channel redirects
75+
// It works like that:
76+
// @TheDailyShow
77+
// -> resolves to thedailyshow
78+
// -> resolves to the id: UCwWhs_6x42TyRM4Wstoq8HA
79+
// Please note that this is not always the case, some handles
80+
// e.g. @google or @Gronkh directly resolve the id
81+
for (int tries = 0;
82+
urlToResolve != null && tries < 3;
83+
tries++) {
84+
final byte[] body = JsonWriter.string(
85+
prepareDesktopJsonBuilder(Localization.DEFAULT, ContentCountry.DEFAULT)
86+
.value("url", urlToResolve)
7187
.done())
7288
.getBytes(StandardCharsets.UTF_8);
7389

74-
final JsonObject jsonResponse = getJsonPostResponse(
90+
final JsonObject jsonResponse = getJsonPostResponse(
7591
"navigation/resolve_url", body, Localization.DEFAULT);
7692

77-
checkIfChannelResponseIsValid(jsonResponse);
93+
checkIfChannelResponseIsValid(jsonResponse);
7894

79-
final JsonObject endpoint = jsonResponse.getObject("endpoint");
95+
endpoint = jsonResponse.getObject("endpoint");
8096

81-
final String webPageType = endpoint.getObject("commandMetadata")
97+
webPageType = endpoint.getObject("commandMetadata")
8298
.getObject("webCommandMetadata")
83-
.getString("webPageType", "");
99+
.getString("webPageType");
100+
101+
urlToResolve = "WEB_PAGE_TYPE_UNKNOWN".equals(webPageType)
102+
? endpoint.getObject("urlEndpoint").getString("url")
103+
: null;
104+
}
84105

85-
final JsonObject browseEndpoint = endpoint.getObject(BROWSE_ENDPOINT);
86-
final String browseId = browseEndpoint.getString(BROWSE_ID, "");
106+
final String browseId = endpoint.getObject(BROWSE_ENDPOINT)
107+
.getString(BROWSE_ID, "");
87108

88-
if (webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_BROWSE")
89-
|| webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_CHANNEL")
109+
if (("WEB_PAGE_TYPE_BROWSE".equalsIgnoreCase(webPageType)
110+
|| "WEB_PAGE_TYPE_CHANNEL".equalsIgnoreCase(webPageType))
90111
&& !browseId.isEmpty()) {
91112
if (!browseId.startsWith("UC")) {
92113
throw new ExtractionException("Redirected id is not pointing to a channel");
93114
}
94115

95116
return browseId;
96117
}
118+
119+
// Otherwise, the code after that will run into an IndexOutOfBoundsException
120+
if (channelId.length < 2) {
121+
throw new ExtractionException("Failed to resolve channelId for " + idOrPath);
122+
}
97123
}
98124

99125
// return the unresolved URL
@@ -175,13 +201,13 @@ public static ChannelResponseData getChannelResponse(@Nonnull final String chann
175201

176202
final String webPageType = endpoint.getObject("commandMetadata")
177203
.getObject("webCommandMetadata")
178-
.getString("webPageType", "");
204+
.getString("webPageType");
179205

180206
final String browseId = endpoint.getObject(BROWSE_ENDPOINT)
181207
.getString(BROWSE_ID, "");
182208

183-
if (webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_BROWSE")
184-
|| webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_CHANNEL")
209+
if (("WEB_PAGE_TYPE_BROWSE".equalsIgnoreCase(webPageType)
210+
|| "WEB_PAGE_TYPE_CHANNEL".equalsIgnoreCase(webPageType))
185211
&& !browseId.isEmpty()) {
186212
if (!browseId.startsWith("UC")) {
187213
throw new ExtractionException("Redirected id is not pointing to a channel");

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemLockupExtractor.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -209,8 +209,10 @@ public String getUploaderUrl() throws ParsingException {
209209
throw new ParsingException("Could not get uploader url");
210210
}
211211

212-
private String resolveUploaderUrlFromRelativeUrl(final String url) throws ParsingException {
213-
return YoutubeChannelLinkHandlerFactory.getInstance().getUrl("c" + url);
212+
private String resolveUploaderUrlFromRelativeUrl(final String relativeUrl)
213+
throws ParsingException {
214+
return YoutubeChannelLinkHandlerFactory.getInstance().getUrl(
215+
relativeUrl.startsWith("/") ? relativeUrl.substring(1) : relativeUrl);
214216
}
215217

216218
@Nonnull

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YouTubeChannelHelperTest.java‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
package org.schabi.newpipe.extractor.services.youtube;
2+
3+
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
4+
import static org.junit.jupiter.api.Assertions.assertEquals;
5+
import static org.junit.jupiter.api.Assertions.assertThrows;
6+
7+
import org.junit.jupiter.api.Test;
8+
import org.junit.jupiter.params.ParameterizedTest;
9+
import org.junit.jupiter.params.provider.ValueSource;
10+
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
11+
12+
class YouTubeChannelHelperTest implements InitYoutubeTest {
13+
14+
@ParameterizedTest
15+
@ValueSource(strings = {
16+
"@TheDailyShow",
17+
"thedailyshow",
18+
"channel/UCwWhs_6x42TyRM4Wstoq8HA",
19+
"UCwWhs_6x42TyRM4Wstoq8HA"
20+
})
21+
void resolveSuccessfulTheDailyShow(final String idOrPath) {
22+
final String id = assertDoesNotThrow(
23+
() -> YoutubeChannelHelper.resolveChannelId(idOrPath));
24+
assertEquals("UCwWhs_6x42TyRM4Wstoq8HA", id);
25+
}
26+
27+
@ParameterizedTest
28+
@ValueSource(strings = {
29+
"@Gronkh",
30+
"gronkh",
31+
"channel/UCYJ61XIK64sp6ZFFS8sctxw",
32+
"UCYJ61XIK64sp6ZFFS8sctxw"
33+
})
34+
void resolveSuccessfulGronkh(final String idOrPath) {
35+
final String id = assertDoesNotThrow(
36+
() -> YoutubeChannelHelper.resolveChannelId(idOrPath));
37+
assertEquals("UCYJ61XIK64sp6ZFFS8sctxw", id);
38+
}
39+
40+
@Test
41+
void resolveFailNonExistingTag() {
42+
assertThrows(ExtractionException.class, () -> YoutubeChannelHelper.resolveChannelId(
43+
"@nonExistingHandleThatWillNeverExist15464"));
44+
}
45+
}

‎extractor/src/test/resources/mocks/v1/org/schabi/newpipe/extractor/services/youtube/youtubechannelhelper/generated_mock_0.json‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
{
2+
"request": {
3+
"httpMethod": "GET",
4+
"url": "https://www.youtube.com/sw.js",
5+
"headers": {
6+
"Referer": [
7+
"https://www.youtube.com"
8+
],
9+
"Origin": [
10+
"https://www.youtube.com"
11+
],
12+
"Accept-Language": [
13+
"en-GB, en;q\u003d0.9"
14+
]
15+
},
16+
"localization": {
17+
"languageCode": "en",
18+
"countryCode": "GB"
19+
}
20+
},
21+
"response": {
22+
"responseCode": 200,
23+
"responseMessage": "",
24+
"responseHeaders": {
25+
"access-control-allow-credentials": [
26+
"true"
27+
],
28+
"access-control-allow-origin": [
29+
"https://www.youtube.com"
30+
],
31+
"alt-svc": [
32+
"h3\u003d\":443\"; ma\u003d2592000,h3-29\u003d\":443\"; ma\u003d2592000"
33+
],
34+
"cache-control": [
35+
"private, max-age\u003d0"
36+
],
37+
"content-security-policy": [
38+
"require-trusted-types-for \u0027script\u0027"
39+
],
40+
"content-security-policy-report-only": [
41+
"script-src \u0027unsafe-eval\u0027 \u0027self\u0027 \u0027unsafe-inline\u0027 https://www.google.com https://apis.google.com https://ssl.gstatic.com https://www.gstatic.com https://www.googletagmanager.com https://www.google-analytics.com https://*.youtube.com https://*.google.com https://*.gstatic.com https://youtube.com https://www.youtube.com https://google.com https://*.doubleclick.net https://*.googleapis.com https://www.googleadservices.com https://tpc.googlesyndication.com https://www.youtubekids.com;report-uri /cspreport/allowlist"
42+
],
43+
"content-type": [
44+
"text/javascript; charset\u003dutf-8"
45+
],
46+
"cross-origin-opener-policy": [
47+
"same-origin; report-to\u003d\"youtube_main\""
48+
],
49+
"date": [
50+
"Thu, 31 Jul 2025 09:08:16 GMT"
51+
],
52+
"document-policy": [
53+
"include-js-call-stacks-in-crash-reports"
54+
],
55+
"expires": [
56+
"Thu, 31 Jul 2025 09:08:16 GMT"
57+
],
58+
"origin-trial": [
59+
"AmhMBR6zCLzDDxpW+HfpP67BqwIknWnyMOXOQGfzYswFmJe+fgaI6XZgAzcxOrzNtP7hEDsOo1jdjFnVr2IdxQ4AAAB4eyJvcmlnaW4iOiJodHRwczovL3lvdXR1YmUuY29tOjQ0MyIsImZlYXR1cmUiOiJXZWJWaWV3WFJlcXVlc3RlZFdpdGhEZXByZWNhdGlvbiIsImV4cGlyeSI6MTc1ODA2NzE5OSwiaXNTdWJkb21haW4iOnRydWV9"
60+
],
61+
"p3p": [
62+
"CP\u003d\"This is not a P3P policy! See http://support.google.com/accounts/answer/151657?hl\u003den-GB for more info.\""
63+
],
64+
"permissions-policy": [
65+
"ch-ua-arch\u003d*, ch-ua-bitness\u003d*, ch-ua-full-version\u003d*, ch-ua-full-version-list\u003d*, ch-ua-model\u003d*, ch-ua-wow64\u003d*, ch-ua-form-factors\u003d*, ch-ua-platform\u003d*, ch-ua-platform-version\u003d*"
66+
],
67+
"report-to": [
68+
"{\"group\":\"youtube_main\",\"max_age\":2592000,\"endpoints\":[{\"url\":\"https://csp.withgoogle.com/csp/report-to/youtube_main\"}]}"
69+
],
70+
"reporting-endpoints": [
71+
"default\u003d\"/web-reports?context\u003deJwNz3tIU3EUB3Bv3zLdnbv3d4RCDRIRimyi0ymWJhQ-Ei2DInGWOt18NKfNuzXLIsIsSaKwUmsFFaRU2gMKZFr9ERiBFT7-yJQeIiFZ1h-ZZGXnjw-Hw5dzOEfnWzEWeUCqXqqRorOd0v2jB6WidS4p43W9lLdKk1ozNWn-lib1-DVpNtIjWaI8UseYR3K890qLU17pmlqwzB5esMzL_IU62O06_OzU4ceEDg1zOlhiZFCsjJEcGbM7ZDzukuF_KGPyswy3RY_vJXokuvWonddjJj8EA50hGB4JwcbdBqS2GtDdZ0D-oAHnWcUfA779NSApTUFvlgKPTUG3Q4HsUXC5RUF8u4IsFnRdweqXCgaiVQSVqNAx80UVw1dUBN9T4RtSMfFLxebfKvaFCmgbBPoTBUrNAkmbBJrSBXx7BLYXC0yyvAqBsmqBGw0CjY2cM-tJga1nBF61cj0rMNcmYLogsOWSwNPbApF3BN71CET0Ciw9EQh6wf2QQOiowPhbgTUzAslzAp8WBCoX2T-BcwEEQzChJozwLJzQF0HoWEvwsakowsB6QkICwcwCzYShFEJpKqE-ndC1jfAokzDOYrIJD9hSDqEwl1DE9rMSZmXlzM4qWTVzMCerYy6mMQ_zssOskV3dSQjYRWhmX_YSci2Ej0U8U0yoqOAdbJEFu_m-I5wfI4SdICywvGbCVyafIqSxptOElS2EgjbC8nb-8ybf1MW_dhM-9BAy_DzTT1AGCaQPOj5993mg-mZ6tAVRxoZat-a22mIP2axGu6vWqRltznJjmatKqyordRSb4kzmuGRTSmx8XHFd3H8KrcfX\""
72+
],
73+
"server": [
74+
"ESF"
75+
],
76+
"set-cookie": [
77+
"YSC\u003dYhEHSiU9_R8; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
78+
"VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dFri, 04-Nov-2022 09:08:16 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone"
79+
],
80+
"strict-transport-security": [
81+
"max-age\u003d31536000"
82+
],
83+
"x-content-type-options": [
84+
"nosniff"
85+
],
86+
"x-frame-options": [
87+
"SAMEORIGIN"
88+
],
89+
"x-xss-protection": [
90+
"0"
91+
]
92+
},
93+
"responseBody": "\n self.addEventListener(\u0027install\u0027, event \u003d\u003e {\n event.waitUntil(self.skipWaiting());\n });\n self.addEventListener(\u0027activate\u0027, event \u003d\u003e {\n event.waitUntil(\n self.clients.claim().then(() \u003d\u003e self.registration.unregister()));\n });\n ",
94+
"latestUrl": "https://www.youtube.com/sw.js"
95+
}
96+
}

‎extractor/src/test/resources/mocks/v1/org/schabi/newpipe/extractor/services/youtube/youtubechannelhelper/generated_mock_1.json‎

Lines changed: 92 additions & 0 deletions
Large diffs are not rendered by default.

0 commit comments

Comments
 (0)