From 943a41fe14cc4538ff4bd27af98891f9b911a4c2 Mon Sep 17 00:00:00 2001 From: h Date: Wed, 22 Apr 2026 16:04:00 +0200 Subject: [PATCH 01/16] Add bullet comments core framework --- .../schabi/newpipe/extractor/InfoItem.java | 3 +- .../newpipe/extractor/StreamingService.java | 19 +++- .../BulletCommentsExtractor.java | 58 ++++++++++++ .../bulletComments/BulletCommentsInfo.java | 91 +++++++++++++++++++ .../BulletCommentsInfoItem.java | 90 ++++++++++++++++++ .../BulletCommentsInfoItemExtractor.java | 57 ++++++++++++ .../BulletCommentsInfoItemsCollector.java | 56 ++++++++++++ 7 files changed, 372 insertions(+), 2 deletions(-) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/bulletComments/BulletCommentsExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/bulletComments/BulletCommentsInfo.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/bulletComments/BulletCommentsInfoItem.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/bulletComments/BulletCommentsInfoItemExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/bulletComments/BulletCommentsInfoItemsCollector.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/InfoItem.java b/extractor/src/main/java/org/schabi/newpipe/extractor/InfoItem.java index 08956b883f..e1fc677aea 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/InfoItem.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/InfoItem.java @@ -76,6 +76,7 @@ public enum InfoType { STREAM, PLAYLIST, CHANNEL, - COMMENT + COMMENT, + BULLET_COMMENT } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java index 0216ba3aa0..1101f5c3f5 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java @@ -2,6 +2,7 @@ import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.channel.tabs.ChannelTabExtractor; +import org.schabi.newpipe.extractor.bulletComments.BulletCommentsExtractor; import org.schabi.newpipe.extractor.comments.CommentsExtractor; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; @@ -76,7 +77,7 @@ public Set getMediaCapabilities() { } public enum MediaCapability { - AUDIO, VIDEO, LIVE, COMMENTS + AUDIO, VIDEO, LIVE, COMMENTS, BULLET_COMMENTS } } @@ -163,6 +164,9 @@ public String toString() { */ public abstract SearchQueryHandlerFactory getSearchQHFactory(); public abstract ListLinkHandlerFactory getCommentsLHFactory(); + public ListLinkHandlerFactory getBulletCommentsLHFactory() { + return null; + } /*////////////////////////////////////////////////////////////////////////// // Extractors @@ -241,6 +245,10 @@ public abstract StreamExtractor getStreamExtractor(LinkHandler linkHandler) public abstract CommentsExtractor getCommentsExtractor(ListLinkHandler linkHandler) throws ExtractionException; + public BulletCommentsExtractor getBulletCommentsExtractor( + final ListLinkHandler linkHandler) throws ExtractionException { + return null; + } /*////////////////////////////////////////////////////////////////////////// // Extractors without link handler @@ -303,6 +311,15 @@ public StreamExtractor getStreamExtractor(final String url) throws ExtractionExc return getStreamExtractor(getStreamLHFactory().fromUrl(url)); } + public BulletCommentsExtractor getBulletCommentsExtractor(final String url) + throws ExtractionException { + final ListLinkHandlerFactory listLinkHandlerFactory = getBulletCommentsLHFactory(); + if (listLinkHandlerFactory == null) { + return null; + } + return getBulletCommentsExtractor(listLinkHandlerFactory.fromUrl(url)); + } + public CommentsExtractor getCommentsExtractor(final String url) throws ExtractionException { final ListLinkHandlerFactory listLinkHandlerFactory = getCommentsLHFactory(); if (listLinkHandlerFactory == null) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/bulletComments/BulletCommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/bulletComments/BulletCommentsExtractor.java new file mode 100644 index 0000000000..d91380df41 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/bulletComments/BulletCommentsExtractor.java @@ -0,0 +1,58 @@ +package org.schabi.newpipe.extractor.bulletComments; + +import javax.annotation.Nonnull; + +import org.schabi.newpipe.extractor.ListExtractor; +import org.schabi.newpipe.extractor.Page; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; + +import java.io.IOException; +import java.util.List; + +public abstract class BulletCommentsExtractor extends ListExtractor { + public BulletCommentsExtractor(final StreamingService service, + final ListLinkHandler uiHandler) { + super(service, uiHandler); + } + + @Nonnull + @Override + public String getName() throws ParsingException { + return "BulletComments"; + } + + @Override + public InfoItemsPage getPage(final Page page) + throws IOException, ExtractionException { + return null; + } + + public List getLiveMessages() throws ParsingException { + return null; + } + + public boolean isLive() { + return false; + } + + public boolean isDisabled() { + return false; + } + + public void disconnect() { + + } + + public void reconnect() { + + } + + public void setCurrentPlayPosition(final long currentPlayPosition) { + } + + public void clearMappingState() { + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/bulletComments/BulletCommentsInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/bulletComments/BulletCommentsInfo.java new file mode 100644 index 0000000000..15ebb61d0b --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/bulletComments/BulletCommentsInfo.java @@ -0,0 +1,91 @@ +package org.schabi.newpipe.extractor.bulletComments; + +import java.io.IOException; + +import org.schabi.newpipe.extractor.ListInfo; +import org.schabi.newpipe.extractor.utils.ExtractorLogger; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.Page; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.utils.ExtractorHelper; + +public final class BulletCommentsInfo extends ListInfo { + private BulletCommentsInfo( + final int serviceId, + final ListLinkHandler listUrlIdHandler, + final String name) { + super(serviceId, listUrlIdHandler, name); + } + + public static BulletCommentsInfo getInfo(final String url) + throws IOException, ExtractionException { + return getInfo(NewPipe.getServiceByUrl(url), url); + } + + public static BulletCommentsInfo getInfo(final StreamingService service, final String url) + throws ExtractionException, IOException { + return getInfo(service.getBulletCommentsExtractor(url)); + } + + public static BulletCommentsInfo getInfo(final BulletCommentsExtractor commentsExtractor) + throws IOException, ExtractionException { + // for services which do not have a comments extractor + if (commentsExtractor == null) { + ExtractorLogger.d("BulletCommentsInfo", "getInfo() extractor is null"); + return null; + } + + ExtractorLogger.d("BulletCommentsInfo", "getInfo() fetching page for " + + commentsExtractor.getUrl()); + commentsExtractor.fetchPage(); + + final String name = commentsExtractor.getName(); + final int serviceId = commentsExtractor.getServiceId(); + final ListLinkHandler listUrlIdHandler = commentsExtractor.getLinkHandler(); + + final BulletCommentsInfo commentsInfo = new BulletCommentsInfo( + serviceId, listUrlIdHandler, name); + commentsInfo.setBulletCommentsExtractor(commentsExtractor); + final InfoItemsPage initialCommentsPage = ExtractorHelper + .getItemsPageOrLogError(commentsInfo, commentsExtractor); + commentsInfo.setRelatedItems(initialCommentsPage.getItems()); + commentsInfo.setNextPage(initialCommentsPage.getNextPage()); + + return commentsInfo; + } + + public static InfoItemsPage getMoreItems( + final BulletCommentsInfo commentsInfo, + final Page page) throws ExtractionException, IOException { + return getMoreItems(NewPipe.getService(commentsInfo.getServiceId()), commentsInfo.getUrl(), + page); + } + + public static InfoItemsPage getMoreItems( + final StreamingService service, + final BulletCommentsInfo commentsInfo, + final Page page) throws IOException, ExtractionException { + return getMoreItems(service, commentsInfo.getUrl(), page); + } + + public static InfoItemsPage getMoreItems( + final StreamingService service, + final String url, + final Page page) throws IOException, ExtractionException { + return service.getBulletCommentsExtractor(url).getPage(page); + } + + private transient BulletCommentsExtractor commentsExtractor; + + public BulletCommentsExtractor getBulletCommentsExtractor() { + return commentsExtractor; + } + + public void setBulletCommentsExtractor( + final BulletCommentsExtractor bulletCommentsExtractor) { + this.commentsExtractor = bulletCommentsExtractor; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/bulletComments/BulletCommentsInfoItem.java b/extractor/src/main/java/org/schabi/newpipe/extractor/bulletComments/BulletCommentsInfoItem.java new file mode 100644 index 0000000000..7639831835 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/bulletComments/BulletCommentsInfoItem.java @@ -0,0 +1,90 @@ +package org.schabi.newpipe.extractor.bulletComments; + +import java.time.Duration; + +import org.schabi.newpipe.extractor.InfoItem; + +import javax.annotation.Nonnull; + +public class BulletCommentsInfoItem extends InfoItem implements Comparable { + @Override + public int compareTo(@Nonnull final BulletCommentsInfoItem bulletCommentsInfoItem) { + return this.duration.compareTo(bulletCommentsInfoItem.duration); + } + + public enum Position { + REGULAR, + BOTTOM, + TOP, + SUPERCHAT + } + + private String commentText; + private int argbColor; + private Position position; + private double relativeFontSize; + /* It really should be named as timePosition or some other thing*/ + private Duration duration; + private int lastingTime; + private boolean isLive; + + public BulletCommentsInfoItem(final int serviceId, final String url, final String name) { + super(InfoType.COMMENT, serviceId, url, name); + } + + public String getCommentText() { + return commentText; + } + + public void setCommentText(final String commentText) { + this.commentText = commentText; + } + + public int getArgbColor() { + return argbColor; + } + + public void setArgbColor(final int argbColor) { + this.argbColor = argbColor; + } + + public Position getPosition() { + return position; + } + + public void setPosition(final Position position) { + this.position = position; + } + + public double getRelativeFontSize() { + return relativeFontSize; + } + + public void setRelativeFontSize(final double relativeFontSize) { + this.relativeFontSize = relativeFontSize; + } + + public Duration getDuration() { + return duration; + } + + public void setDuration(final Duration duration) { + this.duration = duration; + } + + public int getLastingTime() { + return -1; + } + + public void setLastingTime(final int lastingTime) { + this.lastingTime = lastingTime; + } + + public boolean isLive() { + return isLive; + } + + public void setLive(final boolean live) { + isLive = live; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/bulletComments/BulletCommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/bulletComments/BulletCommentsInfoItemExtractor.java new file mode 100644 index 0000000000..b5e273eee3 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/bulletComments/BulletCommentsInfoItemExtractor.java @@ -0,0 +1,57 @@ +package org.schabi.newpipe.extractor.bulletComments; + +import org.schabi.newpipe.extractor.InfoItemExtractor; +import org.schabi.newpipe.extractor.exceptions.ParsingException; + +import java.time.Duration; +import java.util.Collections; +import java.util.List; + +public interface BulletCommentsInfoItemExtractor extends InfoItemExtractor { + @Override + default String getName() throws ParsingException { + return null; + } + + @Override + default List getThumbnails() throws ParsingException { + return Collections.emptyList(); + } + + @Override + default String getUrl() throws ParsingException { + return null; + } + + default String getCommentText() throws ParsingException { + return ""; + } + + /** + * Returns ARGB32 int. White: 0xFFFFFFFF. + * @return ARGB32 int. White: 0xFFFFFFFF. + */ + default int getArgbColor() throws ParsingException { + return 0xFFFFFFFF; + } + + default BulletCommentsInfoItem.Position getPosition() throws ParsingException { + return BulletCommentsInfoItem.Position.REGULAR; + } + + default double getRelativeFontSize() throws ParsingException { + return 0.7; + } + + default int getLastingTime() { + return -1; + } + + default boolean isLive() throws ParsingException { + return false; + } + + // Must be implemented. If that is a live stream you should at least + // calculate the time from the start of the stream. + Duration getDuration() throws ParsingException; +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/bulletComments/BulletCommentsInfoItemsCollector.java b/extractor/src/main/java/org/schabi/newpipe/extractor/bulletComments/BulletCommentsInfoItemsCollector.java new file mode 100644 index 0000000000..65aa328d8f --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/bulletComments/BulletCommentsInfoItemsCollector.java @@ -0,0 +1,56 @@ +package org.schabi.newpipe.extractor.bulletComments; + +import org.schabi.newpipe.extractor.InfoItemsCollector; +import org.schabi.newpipe.extractor.exceptions.ParsingException; + +public class BulletCommentsInfoItemsCollector + extends InfoItemsCollector { + public BulletCommentsInfoItemsCollector(final int serviceId) { + super(serviceId); + } + + @Override + public BulletCommentsInfoItem extract(final BulletCommentsInfoItemExtractor extractor) + throws ParsingException { + final BulletCommentsInfoItem resultItem = new BulletCommentsInfoItem( + getServiceId(), extractor.getUrl(), extractor.getName()); + + // optional information + try { + resultItem.setCommentText(extractor.getCommentText()); + } catch (final Exception e) { + addError(e); + } + try { + resultItem.setArgbColor(extractor.getArgbColor()); + } catch (final Exception e) { + addError(e); + } + try { + resultItem.setPosition(extractor.getPosition()); + } catch (final Exception e) { + addError(e); + } + try { + resultItem.setRelativeFontSize(extractor.getRelativeFontSize()); + } catch (final Exception e) { + addError(e); + } + try { + resultItem.setDuration(extractor.getDuration()); + } catch (final Exception e) { + addError(e); + } + try { + resultItem.setLastingTime(extractor.getLastingTime()); + } catch (final Exception e) { + addError(e); + } + try { + resultItem.setLive(extractor.isLive()); + } catch (final Exception e) { + addError(e); + } + return resultItem; + } +} From 00b933f0d2479ebc5d1425bfb15ca94f4f8dd636 Mon Sep 17 00:00:00 2001 From: h Date: Wed, 22 Apr 2026 16:23:00 +0200 Subject: [PATCH 02/16] Add YouTube bullet comments extractor --- .../services/youtube/WatchDataCache.java | 27 ++ .../youtube/YoutubeBulletCommentPair.java | 21 ++ .../services/youtube/YoutubeService.java | 18 +- .../YoutubeBulletCommentsExtractor.java | 300 ++++++++++++++++++ ...outubeBulletCommentsInfoItemExtractor.java | 62 ++++ .../YoutubeSuperChatInfoItemExtractor.java | 35 ++ ...utubeBulletCommentsLinkHandlerFactory.java | 31 ++ 7 files changed, 493 insertions(+), 1 deletion(-) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/WatchDataCache.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeBulletCommentPair.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeBulletCommentsExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeBulletCommentsInfoItemExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuperChatInfoItemExtractor.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeBulletCommentsLinkHandlerFactory.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/WatchDataCache.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/WatchDataCache.java new file mode 100644 index 0000000000..a0261d2dd0 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/WatchDataCache.java @@ -0,0 +1,27 @@ +package org.schabi.newpipe.extractor.services.youtube; + +import org.schabi.newpipe.extractor.stream.StreamType; + +public class WatchDataCache { + public StreamType streamType; + public long startAt; + public boolean shouldBeLive = true; + public String currentUrl; + // save all the 4 last status + public StreamType lastStreamType; + public long lastStartAt; + public boolean lastShouldBeLive = true; + public String lastCurrentUrl; + // use the url to instance the extractor, then save the current data to lasts + public void init(final String url) { + if (url.equals(currentUrl)) { + return; + } + lastStreamType = streamType; + lastStartAt = startAt; + lastShouldBeLive = shouldBeLive; + lastCurrentUrl = currentUrl; + currentUrl = url; + } + +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeBulletCommentPair.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeBulletCommentPair.java new file mode 100644 index 0000000000..634b524234 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeBulletCommentPair.java @@ -0,0 +1,21 @@ +package org.schabi.newpipe.extractor.services.youtube; + +import com.grack.nanojson.JsonObject; + +public class YoutubeBulletCommentPair { + private final JsonObject data; + // the expected offset of the comment from the start of the video + private final long offsetDuration; + public YoutubeBulletCommentPair(final JsonObject item, final long offsetDuration) { + this.offsetDuration = offsetDuration; + this.data = item; + } + + public JsonObject getData() { + return data; + } + + public long getOffsetDuration() { + return offsetDuration; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java index 7127453fda..a0da76b35a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java @@ -1,6 +1,7 @@ package org.schabi.newpipe.extractor.services.youtube; import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO; +import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.BULLET_COMMENTS; import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.COMMENTS; import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.LIVE; import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.VIDEO; @@ -8,6 +9,7 @@ import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.channel.tabs.ChannelTabExtractor; +import org.schabi.newpipe.extractor.bulletComments.BulletCommentsExtractor; import org.schabi.newpipe.extractor.comments.CommentsExtractor; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.feed.FeedExtractor; @@ -25,6 +27,7 @@ import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelExtractor; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelTabExtractor; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeBulletCommentsExtractor; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeCommentsExtractor; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeFeedExtractor; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMixPlaylistExtractor; @@ -42,6 +45,7 @@ import org.schabi.newpipe.extractor.services.youtube.extractors.kiosk.YoutubeTrendingPodcastsEpisodesExtractor; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelTabLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeBulletCommentsLinkHandlerFactory; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeCommentsLinkHandlerFactory; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeLiveLinkHandlerFactory; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory; @@ -82,9 +86,10 @@ */ public class YoutubeService extends StreamingService { + public WatchDataCache watchDataCache = new WatchDataCache(); public YoutubeService(final int id) { - super(id, "YouTube", EnumSet.of(AUDIO, VIDEO, LIVE, COMMENTS)); + super(id, "YouTube", EnumSet.of(AUDIO, VIDEO, LIVE, COMMENTS, BULLET_COMMENTS)); } @Override @@ -258,6 +263,17 @@ public CommentsExtractor getCommentsExtractor(final ListLinkHandler urlIdHandler return new YoutubeCommentsExtractor(this, urlIdHandler); } + @Override + public ListLinkHandlerFactory getBulletCommentsLHFactory() { + return YoutubeBulletCommentsLinkHandlerFactory.getInstance(); + } + + @Override + public BulletCommentsExtractor getBulletCommentsExtractor(final ListLinkHandler linkHandler) + throws ExtractionException { + return new YoutubeBulletCommentsExtractor(this, linkHandler, watchDataCache); + } + /*////////////////////////////////////////////////////////////////////////// // Localization //////////////////////////////////////////////////////////////////////////*/ diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeBulletCommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeBulletCommentsExtractor.java new file mode 100644 index 0000000000..3bcff12818 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeBulletCommentsExtractor.java @@ -0,0 +1,300 @@ +package org.schabi.newpipe.extractor.services.youtube.extractors; + +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonWriter; + +import org.schabi.newpipe.extractor.Page; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.bulletComments.BulletCommentsExtractor; +import org.schabi.newpipe.extractor.bulletComments.BulletCommentsInfoItem; +import org.schabi.newpipe.extractor.bulletComments.BulletCommentsInfoItemsCollector; +import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.localization.ContentCountry; +import org.schabi.newpipe.extractor.localization.Localization; +import org.schabi.newpipe.extractor.services.youtube.WatchDataCache; +import org.schabi.newpipe.extractor.services.youtube.YoutubeBulletCommentPair; +import org.schabi.newpipe.extractor.stream.StreamType; +import org.schabi.newpipe.extractor.utils.ExtractorLogger; + +import java.io.IOException; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; + +import javax.annotation.Nonnull; + +public class YoutubeBulletCommentsExtractor extends BulletCommentsExtractor { + private static final String TAG = "YTBCExtractor"; + private final boolean shoudldBeLive; + private String lastContinuation; + private ScheduledFuture future; + private boolean disabled = false; + private long currentPlayPosition = 0; + private long lastPlayPosition = 0; + private final boolean isLiveStream; + private final long startTime; + private final String[] continuationKeyTexts = new String[]{ + "timedContinuationData", "invalidationContinuationData" + }; + private final CopyOnWriteArrayList messages = + new CopyOnWriteArrayList<>(); + private final CopyOnWriteArrayList superChatMessages = + new CopyOnWriteArrayList<>(); + private final CopyOnWriteArrayList idList = new CopyOnWriteArrayList<>(); + private boolean shouldSkipFetch = false; + private ScheduledExecutorService executor; + + public YoutubeBulletCommentsExtractor(final StreamingService service, + final ListLinkHandler uiHandler, + final WatchDataCache watchDataCache) + throws ExtractionException { + super(service, uiHandler); + ExtractorLogger.d(TAG, "Constructor called for url=" + uiHandler.getUrl() + + " cacheCurrent=" + watchDataCache.currentUrl + + " cacheLast=" + watchDataCache.lastCurrentUrl + + " startAt=" + watchDataCache.startAt); + if (watchDataCache.currentUrl.equals(uiHandler.getUrl())) { + isLiveStream = watchDataCache.streamType.equals(StreamType.LIVE_STREAM); + startTime = watchDataCache.startAt; + shoudldBeLive = watchDataCache.shouldBeLive; + } else if (watchDataCache.lastCurrentUrl.equals(uiHandler.getUrl())) { + isLiveStream = watchDataCache.lastStreamType.equals(StreamType.LIVE_STREAM); + startTime = watchDataCache.lastStartAt; + shoudldBeLive = watchDataCache.lastShouldBeLive; + } else { + throw new ExtractionException( + "WatchDataCache of current url is not initialized"); + } + } + + @Override + public void onFetchPage(@Nonnull final Downloader downloader) + throws IOException, ExtractionException { + ExtractorLogger.d(TAG, "onFetchPage() called, url=" + getUrl() + + " isLiveStream=" + isLiveStream + " shouldBeLive=" + shoudldBeLive); + final String response = downloader.get(getUrl()).responseBody(); + if (response.contains("Live chat replay is not available") + || response.contains("is disabled") + || (!shoudldBeLive && !isLiveStream)) { + ExtractorLogger.w(TAG, "Live chat disabled for this stream"); + disabled = true; + return; + } + try { + final String ytInitialData = response.split( + Pattern.quote("var ytInitialData = "))[1] + .split(Pattern.quote(";"))[0]; + lastContinuation = JsonParser.object().from(ytInitialData) + .getObject("contents") + .getObject("twoColumnWatchNextResults") + .getObject("conversationBar") + .getObject("liveChatRenderer") + .getArray("continuations") + .getObject(0) + .getObject("reloadContinuationData") + .getString("continuation"); + } catch (final Exception e) { + e.printStackTrace(); + } + } + + private void fetchMessage() { + ExtractorLogger.d(TAG, "fetchMessage() called, lastContinuation=" + + (lastContinuation != null ? "set" : "null") + + " currentPlayPosition=" + currentPlayPosition); + if (shouldSkipFetch) { + shouldSkipFetch = false; + return; + } + if (lastPlayPosition == currentPlayPosition) { + return; // should only happen when watching replay and user pauses + // we do not want to fetch the same message twice + } + if (lastContinuation == null) { + return; + } + try { + final byte[] json = JsonWriter.string(prepareDesktopJsonBuilder( + Localization.DEFAULT, ContentCountry.DEFAULT) + .value("continuation", lastContinuation) + .object("currentPlayerState") + .value("playerOffsetMs", String.valueOf(currentPlayPosition)) + .end() + .done()) + .getBytes(UTF_8); + final JsonObject result; + try { + result = getJsonPostResponse("live_chat/" + + (isLiveStream ? "get_live_chat" : "get_live_chat_replay"), + json, Localization.DEFAULT); + } catch (final Exception e) { + return; + } + + final JsonObject liveChatContinuation = result.getObject("continuationContents") + .getObject("liveChatContinuation"); + final JsonArray temp1 = liveChatContinuation.getArray("continuations"); + final JsonObject lastContinuationParent = temp1.getObject( + (!isLiveStream && temp1.size() == 2) ? 1 : 0); + if (isLiveStream) { + for (final String i : continuationKeyTexts) { + if (lastContinuationParent.has(i)) { + lastContinuation = lastContinuationParent.getObject(i) + .getString("continuation"); + break; + } + if (i.equals(continuationKeyTexts[1])) { + throw new ParsingException( + "Failed to get continuation data"); + } + } + } else { + lastContinuation = lastContinuationParent + .getObject("playerSeekContinuationData") + .getString("continuation"); + if (lastContinuation == null) { + throw new ParsingException("Failed to get continuation data"); + } + } + + lastPlayPosition = currentPlayPosition; + + final JsonArray actions = liveChatContinuation.getArray("actions"); + ExtractorLogger.d(TAG, "fetchMessage() got " + actions.size() + " actions"); + for (int i = 0; i < actions.size(); i++) { + final JsonObject item = isLiveStream + ? actions.getObject(i).getObject("addChatItemAction") + .getObject("item") + : actions.getObject(i).getObject("replayChatItemAction") + .getArray("actions").getObject(0) + .getObject("addChatItemAction") + .getObject("item"); + if (item.has("liveChatTextMessageRenderer")) { + final JsonObject temp = item.getObject("liveChatTextMessageRenderer"); + final String id = temp.getString("id"); + if (!idList.contains(id)) { + messages.add(new YoutubeBulletCommentPair(temp, isLiveStream + ? -1 : Long.parseLong(actions.getObject(i) + .getObject("replayChatItemAction") + .getString("videoOffsetTimeMsec")))); + idList.add(id); + } + } else if (item.has("liveChatPaidMessageRenderer")) { + final JsonObject temp = item.getObject("liveChatPaidMessageRenderer"); + final String id = temp.getString("id"); + if (!idList.contains(id)) { + superChatMessages.add(new YoutubeBulletCommentPair(temp, isLiveStream + ? -1 : Long.parseLong(actions.getObject(i) + .getObject("replayChatItemAction") + .getString("videoOffsetTimeMsec")))); + idList.add(id); + } + } + } + } catch (final Exception e) { + // should never throw any exception as that will stop fetching + } + } + + @Nonnull + @Override + public InfoItemsPage getInitialPage() + throws IOException, ExtractionException { + ExtractorLogger.d(TAG, "getInitialPage() called, disabled=" + isDisabled()); + if (isDisabled()) { + return null; + } + executor = Executors.newSingleThreadScheduledExecutor(); + future = executor.scheduleAtFixedRate(this::fetchMessage, + 1000, 1000, TimeUnit.MILLISECONDS); + ExtractorLogger.d(TAG, "Live chat polling started"); + return null; + } + + @Override + public InfoItemsPage getPage(final Page page) + throws IOException, ExtractionException { + return null; + } + + @Override + public boolean isLive() { + return true; + } + + @Override + public List getLiveMessages() throws ParsingException { + ExtractorLogger.d(TAG, "getLiveMessages() called, messages=" + messages.size() + + " superChats=" + superChatMessages.size()); + final BulletCommentsInfoItemsCollector collector = + new BulletCommentsInfoItemsCollector(getServiceId()); + for (final YoutubeBulletCommentPair item : messages) { + collector.commit(new YoutubeBulletCommentsInfoItemExtractor( + item.getData(), startTime, item.getOffsetDuration())); + } + for (final YoutubeBulletCommentPair item : superChatMessages) { + collector.commit(new YoutubeSuperChatInfoItemExtractor( + item.getData(), startTime, item.getOffsetDuration())); + } + messages.clear(); + superChatMessages.clear(); + return collector.getItems(); + } + + @Override + public void disconnect() { + ExtractorLogger.d(TAG, "disconnect() called"); + if (future != null && !future.isCancelled()) { + future.cancel(true); + } + } + + @Override + public void reconnect() { + ExtractorLogger.d(TAG, "reconnect() called, disabled=" + isDisabled()); + if (!isDisabled() && future != null && future.isCancelled()) { + future = executor.scheduleAtFixedRate(this::fetchMessage, + 1000, 1000, TimeUnit.MILLISECONDS); + ExtractorLogger.d(TAG, "Live chat polling restarted"); + } + } + + @Override + public boolean isDisabled() { + return disabled; + } + + @Override + public void setCurrentPlayPosition(final long currentPlayPosition) { + ExtractorLogger.d(TAG, "setCurrentPlayPosition() called, position=" + + currentPlayPosition); + // 49 is -1 + 50, invalid and shouldn't set position + // or it will causing duplicate messages + if (!this.isLiveStream && currentPlayPosition == 49) { + return; + } + if (this.currentPlayPosition > currentPlayPosition) { + idList.clear(); + shouldSkipFetch = true; + } + this.currentPlayPosition = currentPlayPosition; + } + + @Override + public void clearMappingState() { + idList.clear(); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeBulletCommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeBulletCommentsInfoItemExtractor.java new file mode 100644 index 0000000000..2c710d759b --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeBulletCommentsInfoItemExtractor.java @@ -0,0 +1,62 @@ +package org.schabi.newpipe.extractor.services.youtube.extractors; + +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; + +import org.schabi.newpipe.extractor.bulletComments.BulletCommentsInfoItem; +import org.schabi.newpipe.extractor.bulletComments.BulletCommentsInfoItemExtractor; +import org.schabi.newpipe.extractor.exceptions.ParsingException; + +import java.time.Duration; + +public class YoutubeBulletCommentsInfoItemExtractor implements BulletCommentsInfoItemExtractor { + private final JsonObject data; + private long startTime; + private long offsetDuration; // the expected offset of the comment from the start of the video + public YoutubeBulletCommentsInfoItemExtractor(final JsonObject item, + final long startTime, + final long offsetDuration) { + data = item; + this.startTime = startTime; + this.offsetDuration = offsetDuration; + } + + @Override + public String getCommentText() throws ParsingException { + final JsonArray array = data.getObject("message").getArray("runs"); + final StringBuilder result = new StringBuilder(); + for (int i = 0; i < array.size(); i++) { + if (array.getObject(i).has("text")) { + result.append(array.getObject(i).getString("text")); + } + } + return result.toString().replaceAll("□", ""); + } + + @Override + public int getArgbColor() throws ParsingException { + return BulletCommentsInfoItemExtractor.super.getArgbColor(); + } + + @Override + public BulletCommentsInfoItem.Position getPosition() throws ParsingException { + return BulletCommentsInfoItem.Position.REGULAR; + } + + @Override + public double getRelativeFontSize() throws ParsingException { + return BulletCommentsInfoItemExtractor.super.getRelativeFontSize(); + } + + @Override + public Duration getDuration() throws ParsingException { + // return Duration.ofMillis( + // Long.parseLong(data.getString("timestampUsec")) / 1000 - startTime); + return offsetDuration == -1 ? Duration.ZERO : Duration.ofMillis(offsetDuration); + } + + @Override + public boolean isLive() throws ParsingException { + return true; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuperChatInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuperChatInfoItemExtractor.java new file mode 100644 index 0000000000..1854dd95bc --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuperChatInfoItemExtractor.java @@ -0,0 +1,35 @@ +package org.schabi.newpipe.extractor.services.youtube.extractors; + +import com.grack.nanojson.JsonObject; + +import org.schabi.newpipe.extractor.bulletComments.BulletCommentsInfoItem; +import org.schabi.newpipe.extractor.exceptions.ParsingException; + +public class YoutubeSuperChatInfoItemExtractor extends YoutubeBulletCommentsInfoItemExtractor { + private JsonObject data; + public YoutubeSuperChatInfoItemExtractor(final JsonObject item, final long startTime, + final long offsetDuration) { + super(item, startTime, offsetDuration); + data = item; + } + + @Override + public String getCommentText() throws ParsingException { + final String superResult = super.getCommentText(); + if (superResult.length() == 0) { + return ""; + } + return String.format("(%s) ", data.getObject("purchaseAmountText") + .getString("simpleText")) + super.getCommentText(); + } + + @Override + public int getArgbColor() throws ParsingException { + return (int) data.getLong("bodyBackgroundColor"); + } + + @Override + public BulletCommentsInfoItem.Position getPosition() throws ParsingException { + return BulletCommentsInfoItem.Position.SUPERCHAT; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeBulletCommentsLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeBulletCommentsLinkHandlerFactory.java new file mode 100644 index 0000000000..3fab78a332 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeBulletCommentsLinkHandlerFactory.java @@ -0,0 +1,31 @@ +package org.schabi.newpipe.extractor.services.youtube.linkHandler; + +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; + +import java.util.List; + +public class YoutubeBulletCommentsLinkHandlerFactory extends ListLinkHandlerFactory { + private static final YoutubeBulletCommentsLinkHandlerFactory INSTANCE + = new YoutubeBulletCommentsLinkHandlerFactory(); + + public static YoutubeBulletCommentsLinkHandlerFactory getInstance() { + return INSTANCE; + } + + @Override + public String getId(final String url) throws ParsingException { + return YoutubeStreamLinkHandlerFactory.getInstance().getId(url); + } + + @Override + public boolean onAcceptUrl(final String url) throws ParsingException { + return YoutubeStreamLinkHandlerFactory.getInstance().onAcceptUrl(url); + } + + @Override + public String getUrl(final String id, final List contentFilter, + final String sortFilter) throws ParsingException { + return YoutubeStreamLinkHandlerFactory.getInstance().getUrl(id); + } +} From b012ade8fc5cf15974006a0f746638325173de20 Mon Sep 17 00:00:00 2001 From: h Date: Wed, 22 Apr 2026 17:02:00 +0200 Subject: [PATCH 03/16] Add live chat fallback when comments are disabled --- .../extractors/YoutubeCommentsExtractor.java | 128 +++++++++++++++++- 1 file changed, 125 insertions(+), 3 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java index 8667768a4b..0c3effc270 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java @@ -43,6 +43,16 @@ public class YoutubeCommentsExtractor extends CommentsExtractor { */ private JsonObject ajaxJson; + /** + * Live chat continuation token, used when regular comments are disabled. + */ + private String liveChatContinuation; + + /** + * Whether this video is / was a live stream. + */ + private boolean isLiveStream; + public YoutubeCommentsExtractor( final StreamingService service, final ListLinkHandler uiHandler) { @@ -54,6 +64,10 @@ public YoutubeCommentsExtractor( public InfoItemsPage getInitialPage() throws IOException, ExtractionException { + if (commentsDisabled && liveChatContinuation != null) { + return fetchLiveChat(liveChatContinuation); + } + if (commentsDisabled) { return getInfoItemsPageForDisabledComments(); } @@ -351,10 +365,12 @@ public void onFetchPage(@Nonnull final Downloader downloader) .getBytes(StandardCharsets.UTF_8); // @formatter:on - final String initialToken = - findInitialCommentsToken(getJsonPostResponse("next", body, localization)); + final JsonObject nextResponse = getJsonPostResponse("next", body, localization); + final String initialToken = findInitialCommentsToken(nextResponse); if (initialToken == null) { + // Try to extract live chat continuation for live streams + findLiveChatContinuation(nextResponse); return; } @@ -369,10 +385,116 @@ public void onFetchPage(@Nonnull final Downloader downloader) ajaxJson = getJsonPostResponse("next", ajaxBody, localization); } + /** + * Tries to extract a live chat continuation token from the next response. + * This is used when regular comments are disabled on a live stream. + */ + private void findLiveChatContinuation(final JsonObject nextResponse) { + try { + final JsonObject liveChatRenderer = nextResponse + .getObject("contents") + .getObject("twoColumnWatchNextResults") + .getObject("conversationBar") + .getObject("liveChatRenderer"); + liveChatContinuation = liveChatRenderer + .getArray("continuations") + .getObject(0) + .getObject("reloadContinuationData") + .getString("continuation"); + } catch (final Exception e) { + liveChatContinuation = null; + } + } + + /** + * Fetches live chat messages and converts them to CommentsInfoItem. + */ + private InfoItemsPage fetchLiveChat(final String chatContinuation) + throws IOException, ExtractionException { + final Localization localization = getExtractorLocalization(); + final byte[] json = JsonWriter.string( + prepareDesktopJsonBuilder(localization, getExtractorContentCountry()) + .value("continuation", chatContinuation) + .object("currentPlayerState") + .value("playerOffsetMs", "0") + .end() + .done()) + .getBytes(StandardCharsets.UTF_8); + + final String endpoint = "live_chat/" + (isLiveStream ? "get_live_chat" : "get_live_chat_replay"); + final JsonObject result = getJsonPostResponse(endpoint, json, localization); + + return extractLiveChatComments(result); + } + + /** + * Extracts live chat actions into CommentsInfoItem objects. + */ + private InfoItemsPage extractLiveChatComments( + final JsonObject result) throws ExtractionException { + final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector( + getServiceId()); + + try { + final JsonObject chatContinuation = result + .getObject("continuationContents") + .getObject("liveChatContinuation"); + final JsonArray actions = chatContinuation.getArray("actions"); + + for (int i = 0; i < actions.size(); i++) { + final JsonObject action = actions.getObject(i); + final JsonObject item; + if (action.has("addChatItemAction")) { + item = action.getObject("addChatItemAction") + .getObject("item"); + } else if (action.has("replayChatItemAction")) { + item = action.getObject("replayChatItemAction") + .getArray("actions").getObject(0) + .getObject("addChatItemAction") + .getObject("item"); + } else { + continue; + } + + if (item.has("liveChatTextMessageRenderer")) { + collector.commit(new YoutubeLiveChatInfoItemExtractor( + item.getObject("liveChatTextMessageRenderer"))); + } + } + + // Extract next continuation + final JsonArray continuations = chatContinuation + .getArray("continuations"); + final Page nextPage; + if (!continuations.isEmpty()) { + final JsonObject contObj = continuations.getObject( + continuations.size() - 1); + String nextCont = null; + if (contObj.has("timedContinuationData")) { + nextCont = contObj.getObject("timedContinuationData") + .getString("continuation"); + } else if (contObj.has("invalidationContinuationData")) { + nextCont = contObj.getObject("invalidationContinuationData") + .getString("continuation"); + } else if (contObj.has("liveChatReplayContinuationData")) { + nextCont = contObj.getObject("liveChatReplayContinuationData") + .getString("continuation"); + } + nextPage = nextCont != null ? new Page(getUrl(), nextCont) : null; + } else { + nextPage = null; + } + + return new InfoItemsPage<>(collector, nextPage); + } catch (final Exception e) { + return getInfoItemsPageForDisabledComments(); + } + } + @Override public boolean isCommentsDisabled() { - return commentsDisabled; + return commentsDisabled && liveChatContinuation == null; } @Override From cb82024b8bf461c5b4592bf0a68421a3b39d9ec0 Mon Sep 17 00:00:00 2001 From: h Date: Wed, 22 Apr 2026 17:28:00 +0200 Subject: [PATCH 04/16] Add YoutubeLiveChatInfoItemExtractor --- .../YoutubeLiveChatInfoItemExtractor.java | 103 ++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeLiveChatInfoItemExtractor.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeLiveChatInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeLiveChatInfoItemExtractor.java new file mode 100644 index 0000000000..01ef12079b --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeLiveChatInfoItemExtractor.java @@ -0,0 +1,103 @@ +package org.schabi.newpipe.extractor.services.youtube.extractors; + +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; + +import org.schabi.newpipe.extractor.Image; +import org.schabi.newpipe.extractor.comments.CommentsInfoItemExtractor; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; +import org.schabi.newpipe.extractor.stream.Description; + +import java.util.Collections; +import java.util.List; + +import javax.annotation.Nonnull; + +/** + * Extracts comment info from a YouTube live chat message. + */ +public class YoutubeLiveChatInfoItemExtractor implements CommentsInfoItemExtractor { + + private final JsonObject chatMessage; + + public YoutubeLiveChatInfoItemExtractor(final JsonObject chatMessage) { + this.chatMessage = chatMessage; + } + + @Nonnull + @Override + public Description getCommentText() throws ParsingException { + return new Description(YoutubeParsingHelper.getTextFromObject( + chatMessage.getObject("message")), Description.PLAIN_TEXT); + } + + @Override + public String getCommentId() throws ParsingException { + return chatMessage.getString("id", ""); + } + + @Override + public String getUploaderName() throws ParsingException { + return YoutubeParsingHelper.getTextFromObject( + chatMessage.getObject("authorName")); + } + + @Nonnull + @Override + public List getUploaderAvatars() throws ParsingException { + try { + final JsonArray thumbnails = chatMessage.getObject("authorPhoto") + .getArray("thumbnails"); + return YoutubeParsingHelper.getImagesFromThumbnailsArray(thumbnails); + } catch (final Exception e) { + return List.of(); + } + } + + @Override + public String getUploaderUrl() throws ParsingException { + try { + return YoutubeParsingHelper.getUrlFromNavigationEndpoint( + chatMessage.getObject("authorEndpoint")); + } catch (final Exception e) { + return ""; + } + } + + @Override + public boolean isChannelOwner() throws ParsingException { + final JsonArray badges = chatMessage.getArray("authorBadges"); + for (int i = 0; i < badges.size(); i++) { + final JsonObject badge = badges.getObject(i); + if (badge.has("liveChatAuthorBadgeRenderer")) { + final JsonObject renderer = badge.getObject( + "liveChatAuthorBadgeRenderer"); + if (renderer.has("icon")) { + final String iconType = renderer.getObject("icon") + .getString("iconType", ""); + if ("owner".equals(iconType)) { + return true; + } + } + } + } + return false; + } + + @Nonnull + @Override + public List getThumbnails() throws ParsingException { + return Collections.emptyList(); + } + + @Override + public String getName() throws ParsingException { + return getUploaderName(); + } + + @Override + public String getUrl() throws ParsingException { + return ""; + } +} From 08f6570c8aa849501f34b67814175f34a59fe70b Mon Sep 17 00:00:00 2001 From: h Date: Wed, 22 Apr 2026 17:51:00 +0200 Subject: [PATCH 05/16] Add isLiveChat flag to CommentsExtractor --- .../extractor/comments/CommentsExtractor.java | 8 ++++++++ .../extractor/comments/CommentsInfo.java | 18 ++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsExtractor.java index 1402b1d234..e36c25006f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsExtractor.java @@ -22,6 +22,14 @@ public boolean isCommentsDisabled() throws ExtractionException { return false; } + /** + * @apiNote Warning: This method is experimental and may get removed in a future release. + * @return true if the comments source is a live chat otherwise false (default) + */ + public boolean isLiveChat() throws ExtractionException { + return false; + } + /** * @return the total number of comments */ diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfo.java index f50d6bd998..5dcae1ffc2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfo.java @@ -47,6 +47,7 @@ public static CommentsInfo getInfo(final CommentsExtractor commentsExtractor) final InfoItemsPage initialCommentsPage = ExtractorHelper.getItemsPageOrLogError(commentsInfo, commentsExtractor); commentsInfo.setCommentsDisabled(commentsExtractor.isCommentsDisabled()); + commentsInfo.setLiveChat(commentsExtractor.isLiveChat()); commentsInfo.setRelatedItems(initialCommentsPage.getItems()); try { commentsInfo.setCommentsCount(commentsExtractor.getCommentsCount()); @@ -81,6 +82,7 @@ public static InfoItemsPage getMoreItems( private transient CommentsExtractor commentsExtractor; private boolean commentsDisabled = false; + private boolean liveChat = false; private int commentsCount; public CommentsExtractor getCommentsExtractor() { @@ -106,6 +108,22 @@ public void setCommentsDisabled(final boolean commentsDisabled) { this.commentsDisabled = commentsDisabled; } + /** + * @apiNote Warning: This method is experimental and may get removed in a future release. + * @return {@code true} if the comments are from a live chat otherwise {@code false} (default) + */ + public boolean isLiveChat() { + return liveChat; + } + + /** + * @apiNote Warning: This method is experimental and may get removed in a future release. + * @param liveChat {@code true} if the comments are from a live chat otherwise {@code false} + */ + public void setLiveChat(final boolean liveChat) { + this.liveChat = liveChat; + } + /** * Returns the total number of comments. * From b7829922d72f192df683f6ff6a014091e5b788ba Mon Sep 17 00:00:00 2001 From: h Date: Wed, 22 Apr 2026 18:14:00 +0200 Subject: [PATCH 06/16] Update YoutubeCommentsExtractor for live chat pages --- .../youtube/extractors/YoutubeCommentsExtractor.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java index 0c3effc270..7702fd935c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java @@ -208,6 +208,10 @@ private Page getNextPage(final String continuation) throws ParsingException { public InfoItemsPage getPage(final Page page) throws IOException, ExtractionException { + if (commentsDisabled && liveChatContinuation != null) { + return fetchLiveChat(page.getId()); + } + if (commentsDisabled) { return getInfoItemsPageForDisabledComments(); } @@ -497,6 +501,11 @@ public boolean isCommentsDisabled() { return commentsDisabled && liveChatContinuation == null; } + @Override + public boolean isLiveChat() { + return liveChatContinuation != null; + } + @Override public int getCommentsCount() throws ExtractionException { assertPageFetched(); From 8a7cc399f316e2abc4d7b3b28cd926eeff56753e Mon Sep 17 00:00:00 2001 From: h Date: Thu, 23 Apr 2026 18:12:00 +0200 Subject: [PATCH 07/16] Fix isCommentsDisabled to allow live chat --- .../services/youtube/extractors/YoutubeCommentsExtractor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java index 7702fd935c..4cd6a92b73 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java @@ -498,7 +498,7 @@ private InfoItemsPage extractLiveChatComments( @Override public boolean isCommentsDisabled() { - return commentsDisabled && liveChatContinuation == null; + return commentsDisabled && !isLiveChat(); } @Override From 05a8d3636d751d3541371b916f8c674be0d27571 Mon Sep 17 00:00:00 2001 From: h Date: Thu, 23 Apr 2026 18:34:00 +0200 Subject: [PATCH 08/16] Mark live chat pages with live_chat identifier --- .../youtube/extractors/YoutubeCommentsExtractor.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java index 4cd6a92b73..1b60e2c472 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java @@ -208,7 +208,8 @@ private Page getNextPage(final String continuation) throws ParsingException { public InfoItemsPage getPage(final Page page) throws IOException, ExtractionException { - if (commentsDisabled && liveChatContinuation != null) { + if ("live_chat".equals(page.getUrl()) + || (commentsDisabled && liveChatContinuation != null)) { return fetchLiveChat(page.getId()); } @@ -484,7 +485,7 @@ private InfoItemsPage extractLiveChatComments( nextCont = contObj.getObject("liveChatReplayContinuationData") .getString("continuation"); } - nextPage = nextCont != null ? new Page(getUrl(), nextCont) : null; + nextPage = nextCont != null ? new Page("live_chat", nextCont) : null; } else { nextPage = null; } From 620b756741e309f26bc5a5da83e00a93b92b12eb Mon Sep 17 00:00:00 2001 From: h Date: Thu, 23 Apr 2026 18:55:00 +0200 Subject: [PATCH 09/16] Fix fetchLiveChat using replay endpoint on fresh extractors --- .../services/youtube/extractors/YoutubeCommentsExtractor.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java index 1b60e2c472..54320634fd 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java @@ -210,6 +210,7 @@ public InfoItemsPage getPage(final Page page) if ("live_chat".equals(page.getUrl()) || (commentsDisabled && liveChatContinuation != null)) { + isLiveStream = true; return fetchLiveChat(page.getId()); } @@ -416,6 +417,7 @@ private void findLiveChatContinuation(final JsonObject nextResponse) { */ private InfoItemsPage fetchLiveChat(final String chatContinuation) throws IOException, ExtractionException { + isLiveStream = true; final Localization localization = getExtractorLocalization(); final byte[] json = JsonWriter.string( prepareDesktopJsonBuilder(localization, getExtractorContentCountry()) From 40c54a71900aa5ff618e022e619f6e9ab945654b Mon Sep 17 00:00:00 2001 From: h Date: Thu, 23 Apr 2026 19:18:00 +0200 Subject: [PATCH 10/16] Fix emoji extraction showing null for emoji-only messages --- .../YoutubeLiveChatInfoItemExtractor.java | 37 ++++++++++++++++++- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeLiveChatInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeLiveChatInfoItemExtractor.java index 01ef12079b..6915faa099 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeLiveChatInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeLiveChatInfoItemExtractor.java @@ -28,8 +28,41 @@ public YoutubeLiveChatInfoItemExtractor(final JsonObject chatMessage) { @Nonnull @Override public Description getCommentText() throws ParsingException { - return new Description(YoutubeParsingHelper.getTextFromObject( - chatMessage.getObject("message")), Description.PLAIN_TEXT); + final String text = extractChatMessageText(chatMessage.getObject("message")); + return new Description(text, Description.PLAIN_TEXT); + } + + @Nonnull + private static String extractChatMessageText(final JsonObject message) { + if (message == null || message.isEmpty()) { + return ""; + } + + if (message.has("simpleText")) { + return message.getString("simpleText", ""); + } + + final JsonArray runs = message.getArray("runs"); + if (runs.isEmpty()) { + return ""; + } + + final StringBuilder textBuilder = new StringBuilder(); + for (int i = 0; i < runs.size(); i++) { + final JsonObject run = runs.getObject(i); + if (run.has("text")) { + textBuilder.append(run.getString("text", "")); + } else if (run.has("emoji")) { + final JsonObject emoji = run.getObject("emoji"); + if (emoji.has("emojiId")) { + textBuilder.append(emoji.getString("emojiId", "")); + } else { + textBuilder.append("[emoji]"); + } + } + } + + return textBuilder.toString(); } @Override From 81b39fa4504a1cc309913810d6007cd009d7f751 Mon Sep 17 00:00:00 2001 From: h Date: Thu, 23 Apr 2026 19:42:00 +0200 Subject: [PATCH 11/16] Improve custom emoji fallback handling --- .../YoutubeLiveChatInfoItemExtractor.java | 60 +++++++++++++++++-- 1 file changed, 55 insertions(+), 5 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeLiveChatInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeLiveChatInfoItemExtractor.java index 6915faa099..7c52046984 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeLiveChatInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeLiveChatInfoItemExtractor.java @@ -32,6 +32,11 @@ public Description getCommentText() throws ParsingException { return new Description(text, Description.PLAIN_TEXT); } + /** + * Extracts text from a live chat message, handling both regular text and emojis. + * YouTube live chat messages use {@code runs} array where each element has either + * {@code text} or {@code emoji}. + */ @Nonnull private static String extractChatMessageText(final JsonObject message) { if (message == null || message.isEmpty()) { @@ -51,13 +56,13 @@ private static String extractChatMessageText(final JsonObject message) { for (int i = 0; i < runs.size(); i++) { final JsonObject run = runs.getObject(i); if (run.has("text")) { - textBuilder.append(run.getString("text", "")); + final String text = run.getString("text", ""); + textBuilder.append(text); } else if (run.has("emoji")) { final JsonObject emoji = run.getObject("emoji"); - if (emoji.has("emojiId")) { - textBuilder.append(emoji.getString("emojiId", "")); - } else { - textBuilder.append("[emoji]"); + final String emojiText = extractEmojiText(emoji); + if (emojiText != null) { + textBuilder.append(emojiText); } } } @@ -65,6 +70,51 @@ private static String extractChatMessageText(final JsonObject message) { return textBuilder.toString(); } + /** + * Extracts a textual representation of a YouTube live chat emoji. + * For standard emojis, {@code emojiId} contains the Unicode character. + * For custom emojis, uses the first shortcut (e.g. {@code :wave:}) if available. + */ + @Nonnull + private static String extractEmojiText(final JsonObject emoji) { + if (emoji == null || emoji.isEmpty()) { + return ""; + } + + // For standard emojis, emojiId is the Unicode character itself. + // For custom emojis it is an ID, but still better than nothing. + if (emoji.has("emojiId")) { + final String emojiId = emoji.getString("emojiId", ""); + if (!emojiId.isEmpty()) { + return emojiId; + } + } + + // Try to get shortcuts like ":wave:", ":heart:", ":face-blue-smiling:" + if (emoji.has("shortcuts")) { + final JsonArray shortcuts = emoji.getArray("shortcuts"); + for (int i = 0; i < shortcuts.size(); i++) { + final String shortcut = shortcuts.getString(i, ""); + if (!shortcut.isEmpty()) { + return shortcut; + } + } + } + + // Fallback: try searchTerms + if (emoji.has("searchTerms")) { + final JsonArray searchTerms = emoji.getArray("searchTerms"); + for (int i = 0; i < searchTerms.size(); i++) { + final String term = searchTerms.getString(i, ""); + if (!term.isEmpty()) { + return ":" + term + ":"; + } + } + } + + return "[emoji]"; + } + @Override public String getCommentId() throws ParsingException { return chatMessage.getString("id", ""); From fc533e4967d23738bf71d3fd71375aee8132b04e Mon Sep 17 00:00:00 2001 From: h Date: Thu, 23 Apr 2026 20:08:00 +0200 Subject: [PATCH 12/16] Add debug logging to live chat polling --- .../youtube/extractors/YoutubeCommentsExtractor.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java index 54320634fd..62dc8b4dcc 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java @@ -14,6 +14,7 @@ import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.localization.Localization; import org.schabi.newpipe.extractor.localization.TimeAgoParser; +import org.schabi.newpipe.extractor.utils.ExtractorLogger; import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Utils; @@ -29,6 +30,7 @@ import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; public class YoutubeCommentsExtractor extends CommentsExtractor { + private static final String TAG = YoutubeCommentsExtractor.class.getSimpleName(); private static final String COMMENT_VIEW_MODEL_KEY = "commentViewModel"; private static final String COMMENT_RENDERER_KEY = "commentRenderer"; @@ -65,6 +67,7 @@ public InfoItemsPage getInitialPage() throws IOException, ExtractionException { if (commentsDisabled && liveChatContinuation != null) { + ExtractorLogger.d(TAG, "getInitialPage() routing to live chat"); return fetchLiveChat(liveChatContinuation); } @@ -211,6 +214,7 @@ public InfoItemsPage getPage(final Page page) if ("live_chat".equals(page.getUrl()) || (commentsDisabled && liveChatContinuation != null)) { isLiveStream = true; + ExtractorLogger.d(TAG, "getPage() live chat detected, isLiveStream=true"); return fetchLiveChat(page.getId()); } @@ -418,6 +422,8 @@ private void findLiveChatContinuation(final JsonObject nextResponse) { private InfoItemsPage fetchLiveChat(final String chatContinuation) throws IOException, ExtractionException { isLiveStream = true; + ExtractorLogger.d(TAG, "fetchLiveChat() called with continuation={}", + chatContinuation != null ? chatContinuation.substring(0, Math.min(30, chatContinuation.length())) : "null"); final Localization localization = getExtractorLocalization(); final byte[] json = JsonWriter.string( prepareDesktopJsonBuilder(localization, getExtractorContentCountry()) @@ -429,6 +435,7 @@ private InfoItemsPage fetchLiveChat(final String chatContinuat .getBytes(StandardCharsets.UTF_8); final String endpoint = "live_chat/" + (isLiveStream ? "get_live_chat" : "get_live_chat_replay"); + ExtractorLogger.d(TAG, "fetchLiveChat() using endpoint={} isLiveStream={}", endpoint, isLiveStream); final JsonObject result = getJsonPostResponse(endpoint, json, localization); return extractLiveChatComments(result); @@ -492,8 +499,11 @@ private InfoItemsPage extractLiveChatComments( nextPage = null; } + ExtractorLogger.d(TAG, "extractLiveChatComments() extracted={} nextPage={}", + collector.getItems().size(), nextPage != null); return new InfoItemsPage<>(collector, nextPage); } catch (final Exception e) { + ExtractorLogger.e(TAG, "extractLiveChatComments() failed", e); return getInfoItemsPageForDisabledComments(); } } From 70c27346f64ba506740bf7a83841c838ee173971 Mon Sep 17 00:00:00 2001 From: h Date: Fri, 24 Apr 2026 10:15:00 +0200 Subject: [PATCH 13/16] Fix checkstyle violations in live chat and bullet comments code --- .../bulletComments/BulletCommentsExtractor.java | 2 +- .../extractor/comments/CommentsExtractor.java | 3 ++- .../youtube/extractors/YoutubeCommentsExtractor.java | 12 ++++++++---- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/bulletComments/BulletCommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/bulletComments/BulletCommentsExtractor.java index d91380df41..780207cb8b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/bulletComments/BulletCommentsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/bulletComments/BulletCommentsExtractor.java @@ -49,7 +49,7 @@ public void disconnect() { public void reconnect() { } - + public void setCurrentPlayPosition(final long currentPlayPosition) { } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsExtractor.java index e36c25006f..87f48fc12d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsExtractor.java @@ -24,7 +24,8 @@ public boolean isCommentsDisabled() throws ExtractionException { /** * @apiNote Warning: This method is experimental and may get removed in a future release. - * @return true if the comments source is a live chat otherwise false (default) + * @return true if the comments source is a live chat + * otherwise false (default) */ public boolean isLiveChat() throws ExtractionException { return false; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java index 62dc8b4dcc..5087128c21 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java @@ -422,8 +422,10 @@ private void findLiveChatContinuation(final JsonObject nextResponse) { private InfoItemsPage fetchLiveChat(final String chatContinuation) throws IOException, ExtractionException { isLiveStream = true; - ExtractorLogger.d(TAG, "fetchLiveChat() called with continuation={}", - chatContinuation != null ? chatContinuation.substring(0, Math.min(30, chatContinuation.length())) : "null"); + final String contPreview = chatContinuation != null + ? chatContinuation.substring(0, Math.min(30, chatContinuation.length())) + : "null"; + ExtractorLogger.d(TAG, "fetchLiveChat() called with continuation={}", contPreview); final Localization localization = getExtractorLocalization(); final byte[] json = JsonWriter.string( prepareDesktopJsonBuilder(localization, getExtractorContentCountry()) @@ -434,8 +436,10 @@ private InfoItemsPage fetchLiveChat(final String chatContinuat .done()) .getBytes(StandardCharsets.UTF_8); - final String endpoint = "live_chat/" + (isLiveStream ? "get_live_chat" : "get_live_chat_replay"); - ExtractorLogger.d(TAG, "fetchLiveChat() using endpoint={} isLiveStream={}", endpoint, isLiveStream); + final String endpoint = "live_chat/" + + (isLiveStream ? "get_live_chat" : "get_live_chat_replay"); + ExtractorLogger.d(TAG, "fetchLiveChat() endpoint={} isLiveStream={}", + endpoint, isLiveStream); final JsonObject result = getJsonPostResponse(endpoint, json, localization); return extractLiveChatComments(result); From 9443929655fc7122721ef70cf9d4615150068891 Mon Sep 17 00:00:00 2001 From: h Date: Fri, 24 Apr 2026 10:25:00 +0200 Subject: [PATCH 14/16] Remove debug logging from live chat flow --- .../youtube/extractors/YoutubeCommentsExtractor.java | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java index 5087128c21..87698392e1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java @@ -14,7 +14,6 @@ import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.localization.Localization; import org.schabi.newpipe.extractor.localization.TimeAgoParser; -import org.schabi.newpipe.extractor.utils.ExtractorLogger; import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Utils; @@ -67,7 +66,6 @@ public InfoItemsPage getInitialPage() throws IOException, ExtractionException { if (commentsDisabled && liveChatContinuation != null) { - ExtractorLogger.d(TAG, "getInitialPage() routing to live chat"); return fetchLiveChat(liveChatContinuation); } @@ -214,7 +212,6 @@ public InfoItemsPage getPage(final Page page) if ("live_chat".equals(page.getUrl()) || (commentsDisabled && liveChatContinuation != null)) { isLiveStream = true; - ExtractorLogger.d(TAG, "getPage() live chat detected, isLiveStream=true"); return fetchLiveChat(page.getId()); } @@ -422,10 +419,6 @@ private void findLiveChatContinuation(final JsonObject nextResponse) { private InfoItemsPage fetchLiveChat(final String chatContinuation) throws IOException, ExtractionException { isLiveStream = true; - final String contPreview = chatContinuation != null - ? chatContinuation.substring(0, Math.min(30, chatContinuation.length())) - : "null"; - ExtractorLogger.d(TAG, "fetchLiveChat() called with continuation={}", contPreview); final Localization localization = getExtractorLocalization(); final byte[] json = JsonWriter.string( prepareDesktopJsonBuilder(localization, getExtractorContentCountry()) @@ -438,8 +431,6 @@ private InfoItemsPage fetchLiveChat(final String chatContinuat final String endpoint = "live_chat/" + (isLiveStream ? "get_live_chat" : "get_live_chat_replay"); - ExtractorLogger.d(TAG, "fetchLiveChat() endpoint={} isLiveStream={}", - endpoint, isLiveStream); final JsonObject result = getJsonPostResponse(endpoint, json, localization); return extractLiveChatComments(result); @@ -503,11 +494,8 @@ private InfoItemsPage extractLiveChatComments( nextPage = null; } - ExtractorLogger.d(TAG, "extractLiveChatComments() extracted={} nextPage={}", - collector.getItems().size(), nextPage != null); return new InfoItemsPage<>(collector, nextPage); } catch (final Exception e) { - ExtractorLogger.e(TAG, "extractLiveChatComments() failed", e); return getInfoItemsPageForDisabledComments(); } } From 3169692154a45e38216da4a8cc8424cb99fdb0dd Mon Sep 17 00:00:00 2001 From: h Date: Fri, 24 Apr 2026 10:30:00 +0200 Subject: [PATCH 15/16] Remove unused bullet comments code --- .../schabi/newpipe/extractor/InfoItem.java | 3 +- .../newpipe/extractor/StreamingService.java | 19 +- .../BulletCommentsExtractor.java | 58 ---- .../bulletComments/BulletCommentsInfo.java | 91 ------ .../BulletCommentsInfoItem.java | 90 ------ .../BulletCommentsInfoItemExtractor.java | 57 ---- .../BulletCommentsInfoItemsCollector.java | 56 ---- .../services/youtube/WatchDataCache.java | 27 -- .../youtube/YoutubeBulletCommentPair.java | 21 -- .../services/youtube/YoutubeService.java | 19 +- .../YoutubeBulletCommentsExtractor.java | 300 ------------------ ...outubeBulletCommentsInfoItemExtractor.java | 62 ---- .../YoutubeSuperChatInfoItemExtractor.java | 35 -- ...utubeBulletCommentsLinkHandlerFactory.java | 31 -- 14 files changed, 3 insertions(+), 866 deletions(-) delete mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/bulletComments/BulletCommentsExtractor.java delete mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/bulletComments/BulletCommentsInfo.java delete mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/bulletComments/BulletCommentsInfoItem.java delete mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/bulletComments/BulletCommentsInfoItemExtractor.java delete mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/bulletComments/BulletCommentsInfoItemsCollector.java delete mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/WatchDataCache.java delete mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeBulletCommentPair.java delete mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeBulletCommentsExtractor.java delete mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeBulletCommentsInfoItemExtractor.java delete mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuperChatInfoItemExtractor.java delete mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeBulletCommentsLinkHandlerFactory.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/InfoItem.java b/extractor/src/main/java/org/schabi/newpipe/extractor/InfoItem.java index e1fc677aea..08956b883f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/InfoItem.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/InfoItem.java @@ -76,7 +76,6 @@ public enum InfoType { STREAM, PLAYLIST, CHANNEL, - COMMENT, - BULLET_COMMENT + COMMENT } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java index 1101f5c3f5..0216ba3aa0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java @@ -2,7 +2,6 @@ import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.channel.tabs.ChannelTabExtractor; -import org.schabi.newpipe.extractor.bulletComments.BulletCommentsExtractor; import org.schabi.newpipe.extractor.comments.CommentsExtractor; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; @@ -77,7 +76,7 @@ public Set getMediaCapabilities() { } public enum MediaCapability { - AUDIO, VIDEO, LIVE, COMMENTS, BULLET_COMMENTS + AUDIO, VIDEO, LIVE, COMMENTS } } @@ -164,9 +163,6 @@ public String toString() { */ public abstract SearchQueryHandlerFactory getSearchQHFactory(); public abstract ListLinkHandlerFactory getCommentsLHFactory(); - public ListLinkHandlerFactory getBulletCommentsLHFactory() { - return null; - } /*////////////////////////////////////////////////////////////////////////// // Extractors @@ -245,10 +241,6 @@ public abstract StreamExtractor getStreamExtractor(LinkHandler linkHandler) public abstract CommentsExtractor getCommentsExtractor(ListLinkHandler linkHandler) throws ExtractionException; - public BulletCommentsExtractor getBulletCommentsExtractor( - final ListLinkHandler linkHandler) throws ExtractionException { - return null; - } /*////////////////////////////////////////////////////////////////////////// // Extractors without link handler @@ -311,15 +303,6 @@ public StreamExtractor getStreamExtractor(final String url) throws ExtractionExc return getStreamExtractor(getStreamLHFactory().fromUrl(url)); } - public BulletCommentsExtractor getBulletCommentsExtractor(final String url) - throws ExtractionException { - final ListLinkHandlerFactory listLinkHandlerFactory = getBulletCommentsLHFactory(); - if (listLinkHandlerFactory == null) { - return null; - } - return getBulletCommentsExtractor(listLinkHandlerFactory.fromUrl(url)); - } - public CommentsExtractor getCommentsExtractor(final String url) throws ExtractionException { final ListLinkHandlerFactory listLinkHandlerFactory = getCommentsLHFactory(); if (listLinkHandlerFactory == null) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/bulletComments/BulletCommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/bulletComments/BulletCommentsExtractor.java deleted file mode 100644 index 780207cb8b..0000000000 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/bulletComments/BulletCommentsExtractor.java +++ /dev/null @@ -1,58 +0,0 @@ -package org.schabi.newpipe.extractor.bulletComments; - -import javax.annotation.Nonnull; - -import org.schabi.newpipe.extractor.ListExtractor; -import org.schabi.newpipe.extractor.Page; -import org.schabi.newpipe.extractor.StreamingService; -import org.schabi.newpipe.extractor.exceptions.ExtractionException; -import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; - -import java.io.IOException; -import java.util.List; - -public abstract class BulletCommentsExtractor extends ListExtractor { - public BulletCommentsExtractor(final StreamingService service, - final ListLinkHandler uiHandler) { - super(service, uiHandler); - } - - @Nonnull - @Override - public String getName() throws ParsingException { - return "BulletComments"; - } - - @Override - public InfoItemsPage getPage(final Page page) - throws IOException, ExtractionException { - return null; - } - - public List getLiveMessages() throws ParsingException { - return null; - } - - public boolean isLive() { - return false; - } - - public boolean isDisabled() { - return false; - } - - public void disconnect() { - - } - - public void reconnect() { - - } - - public void setCurrentPlayPosition(final long currentPlayPosition) { - } - - public void clearMappingState() { - } -} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/bulletComments/BulletCommentsInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/bulletComments/BulletCommentsInfo.java deleted file mode 100644 index 15ebb61d0b..0000000000 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/bulletComments/BulletCommentsInfo.java +++ /dev/null @@ -1,91 +0,0 @@ -package org.schabi.newpipe.extractor.bulletComments; - -import java.io.IOException; - -import org.schabi.newpipe.extractor.ListInfo; -import org.schabi.newpipe.extractor.utils.ExtractorLogger; -import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; -import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage; -import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.Page; -import org.schabi.newpipe.extractor.StreamingService; -import org.schabi.newpipe.extractor.exceptions.ExtractionException; -import org.schabi.newpipe.extractor.utils.ExtractorHelper; - -public final class BulletCommentsInfo extends ListInfo { - private BulletCommentsInfo( - final int serviceId, - final ListLinkHandler listUrlIdHandler, - final String name) { - super(serviceId, listUrlIdHandler, name); - } - - public static BulletCommentsInfo getInfo(final String url) - throws IOException, ExtractionException { - return getInfo(NewPipe.getServiceByUrl(url), url); - } - - public static BulletCommentsInfo getInfo(final StreamingService service, final String url) - throws ExtractionException, IOException { - return getInfo(service.getBulletCommentsExtractor(url)); - } - - public static BulletCommentsInfo getInfo(final BulletCommentsExtractor commentsExtractor) - throws IOException, ExtractionException { - // for services which do not have a comments extractor - if (commentsExtractor == null) { - ExtractorLogger.d("BulletCommentsInfo", "getInfo() extractor is null"); - return null; - } - - ExtractorLogger.d("BulletCommentsInfo", "getInfo() fetching page for " - + commentsExtractor.getUrl()); - commentsExtractor.fetchPage(); - - final String name = commentsExtractor.getName(); - final int serviceId = commentsExtractor.getServiceId(); - final ListLinkHandler listUrlIdHandler = commentsExtractor.getLinkHandler(); - - final BulletCommentsInfo commentsInfo = new BulletCommentsInfo( - serviceId, listUrlIdHandler, name); - commentsInfo.setBulletCommentsExtractor(commentsExtractor); - final InfoItemsPage initialCommentsPage = ExtractorHelper - .getItemsPageOrLogError(commentsInfo, commentsExtractor); - commentsInfo.setRelatedItems(initialCommentsPage.getItems()); - commentsInfo.setNextPage(initialCommentsPage.getNextPage()); - - return commentsInfo; - } - - public static InfoItemsPage getMoreItems( - final BulletCommentsInfo commentsInfo, - final Page page) throws ExtractionException, IOException { - return getMoreItems(NewPipe.getService(commentsInfo.getServiceId()), commentsInfo.getUrl(), - page); - } - - public static InfoItemsPage getMoreItems( - final StreamingService service, - final BulletCommentsInfo commentsInfo, - final Page page) throws IOException, ExtractionException { - return getMoreItems(service, commentsInfo.getUrl(), page); - } - - public static InfoItemsPage getMoreItems( - final StreamingService service, - final String url, - final Page page) throws IOException, ExtractionException { - return service.getBulletCommentsExtractor(url).getPage(page); - } - - private transient BulletCommentsExtractor commentsExtractor; - - public BulletCommentsExtractor getBulletCommentsExtractor() { - return commentsExtractor; - } - - public void setBulletCommentsExtractor( - final BulletCommentsExtractor bulletCommentsExtractor) { - this.commentsExtractor = bulletCommentsExtractor; - } -} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/bulletComments/BulletCommentsInfoItem.java b/extractor/src/main/java/org/schabi/newpipe/extractor/bulletComments/BulletCommentsInfoItem.java deleted file mode 100644 index 7639831835..0000000000 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/bulletComments/BulletCommentsInfoItem.java +++ /dev/null @@ -1,90 +0,0 @@ -package org.schabi.newpipe.extractor.bulletComments; - -import java.time.Duration; - -import org.schabi.newpipe.extractor.InfoItem; - -import javax.annotation.Nonnull; - -public class BulletCommentsInfoItem extends InfoItem implements Comparable { - @Override - public int compareTo(@Nonnull final BulletCommentsInfoItem bulletCommentsInfoItem) { - return this.duration.compareTo(bulletCommentsInfoItem.duration); - } - - public enum Position { - REGULAR, - BOTTOM, - TOP, - SUPERCHAT - } - - private String commentText; - private int argbColor; - private Position position; - private double relativeFontSize; - /* It really should be named as timePosition or some other thing*/ - private Duration duration; - private int lastingTime; - private boolean isLive; - - public BulletCommentsInfoItem(final int serviceId, final String url, final String name) { - super(InfoType.COMMENT, serviceId, url, name); - } - - public String getCommentText() { - return commentText; - } - - public void setCommentText(final String commentText) { - this.commentText = commentText; - } - - public int getArgbColor() { - return argbColor; - } - - public void setArgbColor(final int argbColor) { - this.argbColor = argbColor; - } - - public Position getPosition() { - return position; - } - - public void setPosition(final Position position) { - this.position = position; - } - - public double getRelativeFontSize() { - return relativeFontSize; - } - - public void setRelativeFontSize(final double relativeFontSize) { - this.relativeFontSize = relativeFontSize; - } - - public Duration getDuration() { - return duration; - } - - public void setDuration(final Duration duration) { - this.duration = duration; - } - - public int getLastingTime() { - return -1; - } - - public void setLastingTime(final int lastingTime) { - this.lastingTime = lastingTime; - } - - public boolean isLive() { - return isLive; - } - - public void setLive(final boolean live) { - isLive = live; - } -} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/bulletComments/BulletCommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/bulletComments/BulletCommentsInfoItemExtractor.java deleted file mode 100644 index b5e273eee3..0000000000 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/bulletComments/BulletCommentsInfoItemExtractor.java +++ /dev/null @@ -1,57 +0,0 @@ -package org.schabi.newpipe.extractor.bulletComments; - -import org.schabi.newpipe.extractor.InfoItemExtractor; -import org.schabi.newpipe.extractor.exceptions.ParsingException; - -import java.time.Duration; -import java.util.Collections; -import java.util.List; - -public interface BulletCommentsInfoItemExtractor extends InfoItemExtractor { - @Override - default String getName() throws ParsingException { - return null; - } - - @Override - default List getThumbnails() throws ParsingException { - return Collections.emptyList(); - } - - @Override - default String getUrl() throws ParsingException { - return null; - } - - default String getCommentText() throws ParsingException { - return ""; - } - - /** - * Returns ARGB32 int. White: 0xFFFFFFFF. - * @return ARGB32 int. White: 0xFFFFFFFF. - */ - default int getArgbColor() throws ParsingException { - return 0xFFFFFFFF; - } - - default BulletCommentsInfoItem.Position getPosition() throws ParsingException { - return BulletCommentsInfoItem.Position.REGULAR; - } - - default double getRelativeFontSize() throws ParsingException { - return 0.7; - } - - default int getLastingTime() { - return -1; - } - - default boolean isLive() throws ParsingException { - return false; - } - - // Must be implemented. If that is a live stream you should at least - // calculate the time from the start of the stream. - Duration getDuration() throws ParsingException; -} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/bulletComments/BulletCommentsInfoItemsCollector.java b/extractor/src/main/java/org/schabi/newpipe/extractor/bulletComments/BulletCommentsInfoItemsCollector.java deleted file mode 100644 index 65aa328d8f..0000000000 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/bulletComments/BulletCommentsInfoItemsCollector.java +++ /dev/null @@ -1,56 +0,0 @@ -package org.schabi.newpipe.extractor.bulletComments; - -import org.schabi.newpipe.extractor.InfoItemsCollector; -import org.schabi.newpipe.extractor.exceptions.ParsingException; - -public class BulletCommentsInfoItemsCollector - extends InfoItemsCollector { - public BulletCommentsInfoItemsCollector(final int serviceId) { - super(serviceId); - } - - @Override - public BulletCommentsInfoItem extract(final BulletCommentsInfoItemExtractor extractor) - throws ParsingException { - final BulletCommentsInfoItem resultItem = new BulletCommentsInfoItem( - getServiceId(), extractor.getUrl(), extractor.getName()); - - // optional information - try { - resultItem.setCommentText(extractor.getCommentText()); - } catch (final Exception e) { - addError(e); - } - try { - resultItem.setArgbColor(extractor.getArgbColor()); - } catch (final Exception e) { - addError(e); - } - try { - resultItem.setPosition(extractor.getPosition()); - } catch (final Exception e) { - addError(e); - } - try { - resultItem.setRelativeFontSize(extractor.getRelativeFontSize()); - } catch (final Exception e) { - addError(e); - } - try { - resultItem.setDuration(extractor.getDuration()); - } catch (final Exception e) { - addError(e); - } - try { - resultItem.setLastingTime(extractor.getLastingTime()); - } catch (final Exception e) { - addError(e); - } - try { - resultItem.setLive(extractor.isLive()); - } catch (final Exception e) { - addError(e); - } - return resultItem; - } -} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/WatchDataCache.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/WatchDataCache.java deleted file mode 100644 index a0261d2dd0..0000000000 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/WatchDataCache.java +++ /dev/null @@ -1,27 +0,0 @@ -package org.schabi.newpipe.extractor.services.youtube; - -import org.schabi.newpipe.extractor.stream.StreamType; - -public class WatchDataCache { - public StreamType streamType; - public long startAt; - public boolean shouldBeLive = true; - public String currentUrl; - // save all the 4 last status - public StreamType lastStreamType; - public long lastStartAt; - public boolean lastShouldBeLive = true; - public String lastCurrentUrl; - // use the url to instance the extractor, then save the current data to lasts - public void init(final String url) { - if (url.equals(currentUrl)) { - return; - } - lastStreamType = streamType; - lastStartAt = startAt; - lastShouldBeLive = shouldBeLive; - lastCurrentUrl = currentUrl; - currentUrl = url; - } - -} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeBulletCommentPair.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeBulletCommentPair.java deleted file mode 100644 index 634b524234..0000000000 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeBulletCommentPair.java +++ /dev/null @@ -1,21 +0,0 @@ -package org.schabi.newpipe.extractor.services.youtube; - -import com.grack.nanojson.JsonObject; - -public class YoutubeBulletCommentPair { - private final JsonObject data; - // the expected offset of the comment from the start of the video - private final long offsetDuration; - public YoutubeBulletCommentPair(final JsonObject item, final long offsetDuration) { - this.offsetDuration = offsetDuration; - this.data = item; - } - - public JsonObject getData() { - return data; - } - - public long getOffsetDuration() { - return offsetDuration; - } -} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java index a0da76b35a..9bbdea131b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java @@ -1,7 +1,6 @@ package org.schabi.newpipe.extractor.services.youtube; import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO; -import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.BULLET_COMMENTS; import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.COMMENTS; import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.LIVE; import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.VIDEO; @@ -9,7 +8,6 @@ import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.channel.tabs.ChannelTabExtractor; -import org.schabi.newpipe.extractor.bulletComments.BulletCommentsExtractor; import org.schabi.newpipe.extractor.comments.CommentsExtractor; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.feed.FeedExtractor; @@ -27,7 +25,6 @@ import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelExtractor; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelTabExtractor; -import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeBulletCommentsExtractor; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeCommentsExtractor; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeFeedExtractor; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMixPlaylistExtractor; @@ -45,7 +42,6 @@ import org.schabi.newpipe.extractor.services.youtube.extractors.kiosk.YoutubeTrendingPodcastsEpisodesExtractor; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelTabLinkHandlerFactory; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeBulletCommentsLinkHandlerFactory; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeCommentsLinkHandlerFactory; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeLiveLinkHandlerFactory; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory; @@ -86,10 +82,8 @@ */ public class YoutubeService extends StreamingService { - public WatchDataCache watchDataCache = new WatchDataCache(); - public YoutubeService(final int id) { - super(id, "YouTube", EnumSet.of(AUDIO, VIDEO, LIVE, COMMENTS, BULLET_COMMENTS)); + super(id, "YouTube", EnumSet.of(AUDIO, VIDEO, LIVE, COMMENTS)); } @Override @@ -263,17 +257,6 @@ public CommentsExtractor getCommentsExtractor(final ListLinkHandler urlIdHandler return new YoutubeCommentsExtractor(this, urlIdHandler); } - @Override - public ListLinkHandlerFactory getBulletCommentsLHFactory() { - return YoutubeBulletCommentsLinkHandlerFactory.getInstance(); - } - - @Override - public BulletCommentsExtractor getBulletCommentsExtractor(final ListLinkHandler linkHandler) - throws ExtractionException { - return new YoutubeBulletCommentsExtractor(this, linkHandler, watchDataCache); - } - /*////////////////////////////////////////////////////////////////////////// // Localization //////////////////////////////////////////////////////////////////////////*/ diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeBulletCommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeBulletCommentsExtractor.java deleted file mode 100644 index 3bcff12818..0000000000 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeBulletCommentsExtractor.java +++ /dev/null @@ -1,300 +0,0 @@ -package org.schabi.newpipe.extractor.services.youtube.extractors; - -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; -import static java.nio.charset.StandardCharsets.UTF_8; - -import com.grack.nanojson.JsonArray; -import com.grack.nanojson.JsonObject; -import com.grack.nanojson.JsonParser; -import com.grack.nanojson.JsonWriter; - -import org.schabi.newpipe.extractor.Page; -import org.schabi.newpipe.extractor.StreamingService; -import org.schabi.newpipe.extractor.bulletComments.BulletCommentsExtractor; -import org.schabi.newpipe.extractor.bulletComments.BulletCommentsInfoItem; -import org.schabi.newpipe.extractor.bulletComments.BulletCommentsInfoItemsCollector; -import org.schabi.newpipe.extractor.downloader.Downloader; -import org.schabi.newpipe.extractor.exceptions.ExtractionException; -import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; -import org.schabi.newpipe.extractor.localization.ContentCountry; -import org.schabi.newpipe.extractor.localization.Localization; -import org.schabi.newpipe.extractor.services.youtube.WatchDataCache; -import org.schabi.newpipe.extractor.services.youtube.YoutubeBulletCommentPair; -import org.schabi.newpipe.extractor.stream.StreamType; -import org.schabi.newpipe.extractor.utils.ExtractorLogger; - -import java.io.IOException; -import java.util.List; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.TimeUnit; -import java.util.regex.Pattern; - -import javax.annotation.Nonnull; - -public class YoutubeBulletCommentsExtractor extends BulletCommentsExtractor { - private static final String TAG = "YTBCExtractor"; - private final boolean shoudldBeLive; - private String lastContinuation; - private ScheduledFuture future; - private boolean disabled = false; - private long currentPlayPosition = 0; - private long lastPlayPosition = 0; - private final boolean isLiveStream; - private final long startTime; - private final String[] continuationKeyTexts = new String[]{ - "timedContinuationData", "invalidationContinuationData" - }; - private final CopyOnWriteArrayList messages = - new CopyOnWriteArrayList<>(); - private final CopyOnWriteArrayList superChatMessages = - new CopyOnWriteArrayList<>(); - private final CopyOnWriteArrayList idList = new CopyOnWriteArrayList<>(); - private boolean shouldSkipFetch = false; - private ScheduledExecutorService executor; - - public YoutubeBulletCommentsExtractor(final StreamingService service, - final ListLinkHandler uiHandler, - final WatchDataCache watchDataCache) - throws ExtractionException { - super(service, uiHandler); - ExtractorLogger.d(TAG, "Constructor called for url=" + uiHandler.getUrl() - + " cacheCurrent=" + watchDataCache.currentUrl - + " cacheLast=" + watchDataCache.lastCurrentUrl - + " startAt=" + watchDataCache.startAt); - if (watchDataCache.currentUrl.equals(uiHandler.getUrl())) { - isLiveStream = watchDataCache.streamType.equals(StreamType.LIVE_STREAM); - startTime = watchDataCache.startAt; - shoudldBeLive = watchDataCache.shouldBeLive; - } else if (watchDataCache.lastCurrentUrl.equals(uiHandler.getUrl())) { - isLiveStream = watchDataCache.lastStreamType.equals(StreamType.LIVE_STREAM); - startTime = watchDataCache.lastStartAt; - shoudldBeLive = watchDataCache.lastShouldBeLive; - } else { - throw new ExtractionException( - "WatchDataCache of current url is not initialized"); - } - } - - @Override - public void onFetchPage(@Nonnull final Downloader downloader) - throws IOException, ExtractionException { - ExtractorLogger.d(TAG, "onFetchPage() called, url=" + getUrl() - + " isLiveStream=" + isLiveStream + " shouldBeLive=" + shoudldBeLive); - final String response = downloader.get(getUrl()).responseBody(); - if (response.contains("Live chat replay is not available") - || response.contains("is disabled") - || (!shoudldBeLive && !isLiveStream)) { - ExtractorLogger.w(TAG, "Live chat disabled for this stream"); - disabled = true; - return; - } - try { - final String ytInitialData = response.split( - Pattern.quote("var ytInitialData = "))[1] - .split(Pattern.quote(";"))[0]; - lastContinuation = JsonParser.object().from(ytInitialData) - .getObject("contents") - .getObject("twoColumnWatchNextResults") - .getObject("conversationBar") - .getObject("liveChatRenderer") - .getArray("continuations") - .getObject(0) - .getObject("reloadContinuationData") - .getString("continuation"); - } catch (final Exception e) { - e.printStackTrace(); - } - } - - private void fetchMessage() { - ExtractorLogger.d(TAG, "fetchMessage() called, lastContinuation=" - + (lastContinuation != null ? "set" : "null") - + " currentPlayPosition=" + currentPlayPosition); - if (shouldSkipFetch) { - shouldSkipFetch = false; - return; - } - if (lastPlayPosition == currentPlayPosition) { - return; // should only happen when watching replay and user pauses - // we do not want to fetch the same message twice - } - if (lastContinuation == null) { - return; - } - try { - final byte[] json = JsonWriter.string(prepareDesktopJsonBuilder( - Localization.DEFAULT, ContentCountry.DEFAULT) - .value("continuation", lastContinuation) - .object("currentPlayerState") - .value("playerOffsetMs", String.valueOf(currentPlayPosition)) - .end() - .done()) - .getBytes(UTF_8); - final JsonObject result; - try { - result = getJsonPostResponse("live_chat/" - + (isLiveStream ? "get_live_chat" : "get_live_chat_replay"), - json, Localization.DEFAULT); - } catch (final Exception e) { - return; - } - - final JsonObject liveChatContinuation = result.getObject("continuationContents") - .getObject("liveChatContinuation"); - final JsonArray temp1 = liveChatContinuation.getArray("continuations"); - final JsonObject lastContinuationParent = temp1.getObject( - (!isLiveStream && temp1.size() == 2) ? 1 : 0); - if (isLiveStream) { - for (final String i : continuationKeyTexts) { - if (lastContinuationParent.has(i)) { - lastContinuation = lastContinuationParent.getObject(i) - .getString("continuation"); - break; - } - if (i.equals(continuationKeyTexts[1])) { - throw new ParsingException( - "Failed to get continuation data"); - } - } - } else { - lastContinuation = lastContinuationParent - .getObject("playerSeekContinuationData") - .getString("continuation"); - if (lastContinuation == null) { - throw new ParsingException("Failed to get continuation data"); - } - } - - lastPlayPosition = currentPlayPosition; - - final JsonArray actions = liveChatContinuation.getArray("actions"); - ExtractorLogger.d(TAG, "fetchMessage() got " + actions.size() + " actions"); - for (int i = 0; i < actions.size(); i++) { - final JsonObject item = isLiveStream - ? actions.getObject(i).getObject("addChatItemAction") - .getObject("item") - : actions.getObject(i).getObject("replayChatItemAction") - .getArray("actions").getObject(0) - .getObject("addChatItemAction") - .getObject("item"); - if (item.has("liveChatTextMessageRenderer")) { - final JsonObject temp = item.getObject("liveChatTextMessageRenderer"); - final String id = temp.getString("id"); - if (!idList.contains(id)) { - messages.add(new YoutubeBulletCommentPair(temp, isLiveStream - ? -1 : Long.parseLong(actions.getObject(i) - .getObject("replayChatItemAction") - .getString("videoOffsetTimeMsec")))); - idList.add(id); - } - } else if (item.has("liveChatPaidMessageRenderer")) { - final JsonObject temp = item.getObject("liveChatPaidMessageRenderer"); - final String id = temp.getString("id"); - if (!idList.contains(id)) { - superChatMessages.add(new YoutubeBulletCommentPair(temp, isLiveStream - ? -1 : Long.parseLong(actions.getObject(i) - .getObject("replayChatItemAction") - .getString("videoOffsetTimeMsec")))); - idList.add(id); - } - } - } - } catch (final Exception e) { - // should never throw any exception as that will stop fetching - } - } - - @Nonnull - @Override - public InfoItemsPage getInitialPage() - throws IOException, ExtractionException { - ExtractorLogger.d(TAG, "getInitialPage() called, disabled=" + isDisabled()); - if (isDisabled()) { - return null; - } - executor = Executors.newSingleThreadScheduledExecutor(); - future = executor.scheduleAtFixedRate(this::fetchMessage, - 1000, 1000, TimeUnit.MILLISECONDS); - ExtractorLogger.d(TAG, "Live chat polling started"); - return null; - } - - @Override - public InfoItemsPage getPage(final Page page) - throws IOException, ExtractionException { - return null; - } - - @Override - public boolean isLive() { - return true; - } - - @Override - public List getLiveMessages() throws ParsingException { - ExtractorLogger.d(TAG, "getLiveMessages() called, messages=" + messages.size() - + " superChats=" + superChatMessages.size()); - final BulletCommentsInfoItemsCollector collector = - new BulletCommentsInfoItemsCollector(getServiceId()); - for (final YoutubeBulletCommentPair item : messages) { - collector.commit(new YoutubeBulletCommentsInfoItemExtractor( - item.getData(), startTime, item.getOffsetDuration())); - } - for (final YoutubeBulletCommentPair item : superChatMessages) { - collector.commit(new YoutubeSuperChatInfoItemExtractor( - item.getData(), startTime, item.getOffsetDuration())); - } - messages.clear(); - superChatMessages.clear(); - return collector.getItems(); - } - - @Override - public void disconnect() { - ExtractorLogger.d(TAG, "disconnect() called"); - if (future != null && !future.isCancelled()) { - future.cancel(true); - } - } - - @Override - public void reconnect() { - ExtractorLogger.d(TAG, "reconnect() called, disabled=" + isDisabled()); - if (!isDisabled() && future != null && future.isCancelled()) { - future = executor.scheduleAtFixedRate(this::fetchMessage, - 1000, 1000, TimeUnit.MILLISECONDS); - ExtractorLogger.d(TAG, "Live chat polling restarted"); - } - } - - @Override - public boolean isDisabled() { - return disabled; - } - - @Override - public void setCurrentPlayPosition(final long currentPlayPosition) { - ExtractorLogger.d(TAG, "setCurrentPlayPosition() called, position=" - + currentPlayPosition); - // 49 is -1 + 50, invalid and shouldn't set position - // or it will causing duplicate messages - if (!this.isLiveStream && currentPlayPosition == 49) { - return; - } - if (this.currentPlayPosition > currentPlayPosition) { - idList.clear(); - shouldSkipFetch = true; - } - this.currentPlayPosition = currentPlayPosition; - } - - @Override - public void clearMappingState() { - idList.clear(); - } -} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeBulletCommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeBulletCommentsInfoItemExtractor.java deleted file mode 100644 index 2c710d759b..0000000000 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeBulletCommentsInfoItemExtractor.java +++ /dev/null @@ -1,62 +0,0 @@ -package org.schabi.newpipe.extractor.services.youtube.extractors; - -import com.grack.nanojson.JsonArray; -import com.grack.nanojson.JsonObject; - -import org.schabi.newpipe.extractor.bulletComments.BulletCommentsInfoItem; -import org.schabi.newpipe.extractor.bulletComments.BulletCommentsInfoItemExtractor; -import org.schabi.newpipe.extractor.exceptions.ParsingException; - -import java.time.Duration; - -public class YoutubeBulletCommentsInfoItemExtractor implements BulletCommentsInfoItemExtractor { - private final JsonObject data; - private long startTime; - private long offsetDuration; // the expected offset of the comment from the start of the video - public YoutubeBulletCommentsInfoItemExtractor(final JsonObject item, - final long startTime, - final long offsetDuration) { - data = item; - this.startTime = startTime; - this.offsetDuration = offsetDuration; - } - - @Override - public String getCommentText() throws ParsingException { - final JsonArray array = data.getObject("message").getArray("runs"); - final StringBuilder result = new StringBuilder(); - for (int i = 0; i < array.size(); i++) { - if (array.getObject(i).has("text")) { - result.append(array.getObject(i).getString("text")); - } - } - return result.toString().replaceAll("□", ""); - } - - @Override - public int getArgbColor() throws ParsingException { - return BulletCommentsInfoItemExtractor.super.getArgbColor(); - } - - @Override - public BulletCommentsInfoItem.Position getPosition() throws ParsingException { - return BulletCommentsInfoItem.Position.REGULAR; - } - - @Override - public double getRelativeFontSize() throws ParsingException { - return BulletCommentsInfoItemExtractor.super.getRelativeFontSize(); - } - - @Override - public Duration getDuration() throws ParsingException { - // return Duration.ofMillis( - // Long.parseLong(data.getString("timestampUsec")) / 1000 - startTime); - return offsetDuration == -1 ? Duration.ZERO : Duration.ofMillis(offsetDuration); - } - - @Override - public boolean isLive() throws ParsingException { - return true; - } -} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuperChatInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuperChatInfoItemExtractor.java deleted file mode 100644 index 1854dd95bc..0000000000 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuperChatInfoItemExtractor.java +++ /dev/null @@ -1,35 +0,0 @@ -package org.schabi.newpipe.extractor.services.youtube.extractors; - -import com.grack.nanojson.JsonObject; - -import org.schabi.newpipe.extractor.bulletComments.BulletCommentsInfoItem; -import org.schabi.newpipe.extractor.exceptions.ParsingException; - -public class YoutubeSuperChatInfoItemExtractor extends YoutubeBulletCommentsInfoItemExtractor { - private JsonObject data; - public YoutubeSuperChatInfoItemExtractor(final JsonObject item, final long startTime, - final long offsetDuration) { - super(item, startTime, offsetDuration); - data = item; - } - - @Override - public String getCommentText() throws ParsingException { - final String superResult = super.getCommentText(); - if (superResult.length() == 0) { - return ""; - } - return String.format("(%s) ", data.getObject("purchaseAmountText") - .getString("simpleText")) + super.getCommentText(); - } - - @Override - public int getArgbColor() throws ParsingException { - return (int) data.getLong("bodyBackgroundColor"); - } - - @Override - public BulletCommentsInfoItem.Position getPosition() throws ParsingException { - return BulletCommentsInfoItem.Position.SUPERCHAT; - } -} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeBulletCommentsLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeBulletCommentsLinkHandlerFactory.java deleted file mode 100644 index 3fab78a332..0000000000 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeBulletCommentsLinkHandlerFactory.java +++ /dev/null @@ -1,31 +0,0 @@ -package org.schabi.newpipe.extractor.services.youtube.linkHandler; - -import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; - -import java.util.List; - -public class YoutubeBulletCommentsLinkHandlerFactory extends ListLinkHandlerFactory { - private static final YoutubeBulletCommentsLinkHandlerFactory INSTANCE - = new YoutubeBulletCommentsLinkHandlerFactory(); - - public static YoutubeBulletCommentsLinkHandlerFactory getInstance() { - return INSTANCE; - } - - @Override - public String getId(final String url) throws ParsingException { - return YoutubeStreamLinkHandlerFactory.getInstance().getId(url); - } - - @Override - public boolean onAcceptUrl(final String url) throws ParsingException { - return YoutubeStreamLinkHandlerFactory.getInstance().onAcceptUrl(url); - } - - @Override - public String getUrl(final String id, final List contentFilter, - final String sortFilter) throws ParsingException { - return YoutubeStreamLinkHandlerFactory.getInstance().getUrl(id); - } -} From 05f6a5e386d9fce65477c7e9210cff33522f2615 Mon Sep 17 00:00:00 2001 From: h Date: Fri, 24 Apr 2026 11:00:00 +0200 Subject: [PATCH 16/16] Separate live chat from comments extractor --- .../extractor/comments/CommentsExtractor.java | 8 ++++++ .../extractors/YoutubeCommentsExtractor.java | 28 ++++--------------- .../extractors/YoutubeStreamExtractor.java | 20 +++++++++++++ .../extractor/stream/StreamExtractor.java | 16 +++++++++++ .../newpipe/extractor/stream/StreamInfo.java | 24 ++++++++++++++++ 5 files changed, 74 insertions(+), 22 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsExtractor.java index 87f48fc12d..c5de3f4bf9 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsExtractor.java @@ -31,6 +31,14 @@ public boolean isLiveChat() throws ExtractionException { return false; } + /** + * @apiNote Warning: This method is experimental and may get removed in a future release. + * Configures this extractor to fetch live chat messages using the given continuation. + */ + public void setLiveChatContinuation(final String continuation) { + // no-op by default + } + /** * @return the total number of comments */ diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java index 87698392e1..23f5c1f729 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java @@ -65,7 +65,7 @@ public YoutubeCommentsExtractor( public InfoItemsPage getInitialPage() throws IOException, ExtractionException { - if (commentsDisabled && liveChatContinuation != null) { + if (liveChatContinuation != null) { return fetchLiveChat(liveChatContinuation); } @@ -209,8 +209,7 @@ private Page getNextPage(final String continuation) throws ParsingException { public InfoItemsPage getPage(final Page page) throws IOException, ExtractionException { - if ("live_chat".equals(page.getUrl()) - || (commentsDisabled && liveChatContinuation != null)) { + if ("live_chat".equals(page.getUrl()) || liveChatContinuation != null) { isLiveStream = true; return fetchLiveChat(page.getId()); } @@ -376,8 +375,6 @@ public void onFetchPage(@Nonnull final Downloader downloader) final String initialToken = findInitialCommentsToken(nextResponse); if (initialToken == null) { - // Try to extract live chat continuation for live streams - findLiveChatContinuation(nextResponse); return; } @@ -393,24 +390,11 @@ public void onFetchPage(@Nonnull final Downloader downloader) } /** - * Tries to extract a live chat continuation token from the next response. - * This is used when regular comments are disabled on a live stream. + * Configures this extractor to fetch live chat messages. */ - private void findLiveChatContinuation(final JsonObject nextResponse) { - try { - final JsonObject liveChatRenderer = nextResponse - .getObject("contents") - .getObject("twoColumnWatchNextResults") - .getObject("conversationBar") - .getObject("liveChatRenderer"); - liveChatContinuation = liveChatRenderer - .getArray("continuations") - .getObject(0) - .getObject("reloadContinuationData") - .getString("continuation"); - } catch (final Exception e) { - liveChatContinuation = null; - } + @Override + public void setLiveChatContinuation(final String continuation) { + this.liveChatContinuation = continuation; } /** diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 0fa6ac21d8..d204d87b70 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -879,6 +879,26 @@ public void onFetchPage(@Nonnull final Downloader downloader) .done()) .getBytes(StandardCharsets.UTF_8); nextResponse = getJsonPostResponse(NEXT, nextBody, localization); + + // Check for live chat availability + findLiveChatContinuation(nextResponse); + } + + private void findLiveChatContinuation(final JsonObject response) { + try { + final JsonObject liveChatRenderer = response + .getObject("contents") + .getObject("twoColumnWatchNextResults") + .getObject("conversationBar") + .getObject("liveChatRenderer"); + liveChatContinuation = liveChatRenderer + .getArray("continuations") + .getObject(0) + .getObject("reloadContinuationData") + .getString("continuation"); + } catch (final Exception e) { + liveChatContinuation = null; + } } private static void checkPlayabilityStatus(@Nonnull final JsonObject playabilityStatus) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java index 63650a7906..a8774c7bf0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java @@ -56,6 +56,22 @@ public StreamExtractor(final StreamingService service, final LinkHandler linkHan super(service, linkHandler); } + /** + * @return {@code true} if this stream has a live chat available + */ + public boolean hasLiveChat() { + return liveChatContinuation != null; + } + + /** + * @return the live chat continuation token, or {@code null} if not available + */ + public String getLiveChatContinuation() { + return liveChatContinuation; + } + + protected String liveChatContinuation = null; + /** * The original textual date provided by the service. Should be used as a fallback if * {@link #getUploadDate()} isn't provided by the service, or it fails for some reason. diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java index 9a0ea72f61..80f5a36a1d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java @@ -353,6 +353,12 @@ private static void extractOptionalData(final StreamInfo streamInfo, } catch (final Exception e) { streamInfo.addError(e); } + try { + streamInfo.setLiveChat(extractor.hasLiveChat()); + streamInfo.setLiveChatContinuation(extractor.getLiveChatContinuation()); + } catch (final Exception e) { + streamInfo.addError(e); + } streamInfo.setRelatedItems(ExtractorHelper.getRelatedItemsOrLogError(streamInfo, extractor)); @@ -406,6 +412,8 @@ private static void extractOptionalData(final StreamInfo streamInfo, private boolean shortFormContent = false; @Nonnull private ContentAvailability contentAvailability = ContentAvailability.AVAILABLE; + private boolean liveChat = false; + private String liveChatContinuation = null; /** * Preview frames, e.g. for the storyboard / seekbar thumbnail preview @@ -761,4 +769,20 @@ public ContentAvailability getContentAvailability() { public void setContentAvailability(@Nonnull final ContentAvailability availability) { this.contentAvailability = availability; } + + public boolean hasLiveChat() { + return liveChat; + } + + public void setLiveChat(final boolean liveChat) { + this.liveChat = liveChat; + } + + public String getLiveChatContinuation() { + return liveChatContinuation; + } + + public void setLiveChatContinuation(final String liveChatContinuation) { + this.liveChatContinuation = liveChatContinuation; + } }