Skip to content

Commit 4c987a5

Browse files
wb9688wb9688
committed
Support YouTube's new continuations for search
1 parent df28a08 commit 4c987a5

2 files changed

Lines changed: 98 additions & 16 deletions

File tree

‎extractor/src/main/java/org/schabi/newpipe/extractor/Page.java‎

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,35 +8,45 @@
88

99
public class Page implements Serializable {
1010
private final String url;
11+
private final String id;
1112
private final List<String> ids;
1213
private final Map<String, String> cookies;
1314

14-
public Page(final String url, final List<String> ids, final Map<String, String> cookies) {
15+
public Page(final String url, final String id, final List<String> ids, final Map<String, String> cookies) {
1516
this.url = url;
17+
this.id = id;
1618
this.ids = ids;
1719
this.cookies = cookies;
1820
}
1921

2022
public Page(final String url) {
21-
this(url, null, null);
23+
this(url, null, null, null);
24+
}
25+
26+
public Page(final String url, final String id) {
27+
this(url, id, null, null);
2228
}
2329

2430
public Page(final String url, final Map<String, String> cookies) {
25-
this(url, null, cookies);
31+
this(url, null, null, cookies);
2632
}
2733

2834
public Page(final List<String> ids) {
29-
this(null, ids, null);
35+
this(null, null, ids, null);
3036
}
3137

3238
public Page(final List<String> ids, final Map<String, String> cookies) {
33-
this(null, ids, cookies);
39+
this(null, null, ids, cookies);
3440
}
3541

3642
public String getUrl() {
3743
return url;
3844
}
3945

46+
public String getId() {
47+
return id;
48+
}
49+
4050
public List<String> getIds() {
4151
return ids;
4252
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java‎

Lines changed: 83 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,10 @@
22

33
import com.grack.nanojson.JsonArray;
44
import com.grack.nanojson.JsonObject;
5+
import com.grack.nanojson.JsonParser;
6+
import com.grack.nanojson.JsonParserException;
7+
import com.grack.nanojson.JsonWriter;
8+
59
import org.schabi.newpipe.extractor.InfoItem;
610
import org.schabi.newpipe.extractor.Page;
711
import org.schabi.newpipe.extractor.StreamingService;
@@ -14,11 +18,18 @@
1418
import org.schabi.newpipe.extractor.search.SearchExtractor;
1519
import org.schabi.newpipe.extractor.utils.JsonUtils;
1620

17-
import javax.annotation.Nonnull;
1821
import java.io.IOException;
22+
import java.util.Collections;
23+
import java.util.HashMap;
24+
import java.util.List;
25+
import java.util.Map;
1926

27+
import javax.annotation.Nonnull;
28+
29+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getClientVersion;
2030
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse;
2131
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
32+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
2233
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
2334

2435
/*
@@ -104,12 +115,16 @@ public InfoItemsPage<InfoItem> getInitialPage() throws ExtractionException {
104115

105116
Page nextPage = null;
106117

107-
for (Object section : sections) {
108-
final JsonObject itemSectionRenderer = ((JsonObject) section).getObject("itemSectionRenderer");
118+
for (final Object section : sections) {
119+
if (((JsonObject) section).has("itemSectionRenderer")) {
120+
final JsonObject itemSectionRenderer = ((JsonObject) section).getObject("itemSectionRenderer");
109121

110-
collectStreamsFrom(collector, itemSectionRenderer.getArray("contents"));
122+
collectStreamsFrom(collector, itemSectionRenderer.getArray("contents"));
111123

112-
nextPage = getNextPageFrom(itemSectionRenderer.getArray("continuations"));
124+
nextPage = getNextPageFrom(itemSectionRenderer.getArray("continuations"));
125+
} else if (((JsonObject) section).has("continuationItemRenderer")) {
126+
nextPage = getNewNextPageFrom(((JsonObject) section).getObject("continuationItemRenderer"));
127+
}
113128
}
114129

115130
return new InfoItemsPage<>(collector, nextPage);
@@ -122,15 +137,58 @@ public InfoItemsPage<InfoItem> getPage(final Page page) throws IOException, Extr
122137
}
123138

124139
final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId());
125-
final JsonArray ajaxJson = getJsonResponse(page.getUrl(), getExtractorLocalization());
126140

127-
final JsonObject itemSectionRenderer = ajaxJson.getObject(1).getObject("response")
128-
.getObject("continuationContents").getObject("itemSectionContinuation");
141+
if (page.getId() == null) {
142+
final JsonArray ajaxJson = getJsonResponse(page.getUrl(), getExtractorLocalization());
143+
144+
final JsonObject itemSectionContinuation = ajaxJson.getObject(1).getObject("response")
145+
.getObject("continuationContents").getObject("itemSectionContinuation");
146+
147+
collectStreamsFrom(collector, itemSectionContinuation.getArray("contents"));
148+
final JsonArray continuations = itemSectionContinuation.getArray("continuations");
149+
150+
return new InfoItemsPage<>(collector, getNextPageFrom(continuations));
151+
} else {
152+
// @formatter:off
153+
final byte[] json = JsonWriter.string()
154+
.object()
155+
.object("context")
156+
.object("client")
157+
.value("hl", "en")
158+
.value("gl", getExtractorContentCountry().getCountryCode())
159+
.value("clientName", "WEB")
160+
.value("clientVersion", getClientVersion())
161+
.value("utcOffsetMinutes", 0)
162+
.end()
163+
.object("request").end()
164+
.object("user").end()
165+
.end()
166+
.value("continuation", page.getId())
167+
.end().done().getBytes("UTF-8");
168+
// @formatter:on
169+
170+
final Map<String, List<String>> headers = new HashMap<>();
171+
headers.put("Origin", Collections.singletonList("https://www.youtube.com"));
172+
headers.put("Referer", Collections.singletonList(this.getUrl()));
173+
headers.put("Content-Type", Collections.singletonList("application/json"));
174+
175+
final String responseBody = getValidJsonResponseBody(getDownloader().post(page.getUrl(), headers, json));
176+
177+
final JsonObject ajaxJson;
178+
try {
179+
ajaxJson = JsonParser.object().from(responseBody);
180+
} catch (JsonParserException e) {
181+
throw new ParsingException("Could not parse JSON", e);
182+
}
129183

130-
collectStreamsFrom(collector, itemSectionRenderer.getArray("contents"));
131-
final JsonArray continuations = itemSectionRenderer.getArray("continuations");
184+
final JsonArray continuationItems = ajaxJson.getArray("onResponseReceivedCommands")
185+
.getObject(0).getObject("appendContinuationItemsAction").getArray("continuationItems");
132186

133-
return new InfoItemsPage<>(collector, getNextPageFrom(continuations));
187+
final JsonArray contents = continuationItems.getObject(0).getObject("itemSectionRenderer").getArray("contents");
188+
collectStreamsFrom(collector, contents);
189+
190+
return new InfoItemsPage<>(collector, getNewNextPageFrom(continuationItems.getObject(1).getObject("continuationItemRenderer")));
191+
}
134192
}
135193

136194
private void collectStreamsFrom(final InfoItemsSearchCollector collector, final JsonArray videos) throws NothingFoundException, ParsingException {
@@ -162,4 +220,18 @@ private Page getNextPageFrom(final JsonArray continuations) throws ParsingExcept
162220
return new Page(getUrl() + "&pbj=1&ctoken=" + continuation + "&continuation=" + continuation
163221
+ "&itct=" + clickTrackingParams);
164222
}
223+
224+
private Page getNewNextPageFrom(final JsonObject continuationItemRenderer) {
225+
if (isNullOrEmpty(continuationItemRenderer)) {
226+
return null;
227+
}
228+
229+
final String token = continuationItemRenderer.getObject("continuationEndpoint")
230+
.getObject("continuationCommand").getString("token");
231+
232+
// FIXME: Key needs to be extracted
233+
final String url = "https://www.youtube.com/youtubei/v1/search?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8";
234+
235+
return new Page(url, token);
236+
}
165237
}

0 commit comments

Comments
 (0)