Feature/live chat comments

extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsExtractor.java

@@ -22,6 +22,15 @@ public boolean isCommentsDisabled() throws ExtractionException {
         return false;
     }
 
+    /**
+     * @apiNote Warning: This method is experimental and may get removed in a future release.
+     * @return <code>true</code> if the comments source is a live chat
+     * otherwise <code>false</code> (default)
+     */
+    public boolean isLiveChat() throws ExtractionException {
+        return false;
+    }
+
     /**
      * @return the total number of comments
      */

extractor/src/main/java/org/schabi/newpipe/extractor/comments/CommentsInfo.java

@@ -47,6 +47,7 @@ public static CommentsInfo getInfo(final CommentsExtractor commentsExtractor)
         final InfoItemsPage<CommentsInfoItem> initialCommentsPage =
                 ExtractorHelper.getItemsPageOrLogError(commentsInfo, commentsExtractor);
         commentsInfo.setCommentsDisabled(commentsExtractor.isCommentsDisabled());
+        commentsInfo.setLiveChat(commentsExtractor.isLiveChat());
         commentsInfo.setRelatedItems(initialCommentsPage.getItems());
         try {
             commentsInfo.setCommentsCount(commentsExtractor.getCommentsCount());
@@ -81,6 +82,7 @@ public static InfoItemsPage<CommentsInfoItem> getMoreItems(
 
     private transient CommentsExtractor commentsExtractor;
     private boolean commentsDisabled = false;
+    private boolean liveChat = false;
     private int commentsCount;
 
     public CommentsExtractor getCommentsExtractor() {
@@ -106,6 +108,22 @@ public void setCommentsDisabled(final boolean commentsDisabled) {
         this.commentsDisabled = commentsDisabled;
     }
 
+    /**
+     * @apiNote Warning: This method is experimental and may get removed in a future release.
+     * @return {@code true} if the comments are from a live chat otherwise {@code false} (default)
+     */
+    public boolean isLiveChat() {
+        return liveChat;
+    }
+
+    /**
+     * @apiNote Warning: This method is experimental and may get removed in a future release.
+     * @param liveChat {@code true} if the comments are from a live chat otherwise {@code false}
+     */
+    public void setLiveChat(final boolean liveChat) {
+        this.liveChat = liveChat;
+    }
+
     /**
      * Returns the total number of comments.
      *

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java

@@ -82,7 +82,6 @@
  */
 
 public class YoutubeService extends StreamingService {
-
     public YoutubeService(final int id) {
         super(id, "YouTube", EnumSet.of(AUDIO, VIDEO, LIVE, COMMENTS));
     }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java

@@ -29,6 +29,7 @@
 import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
 
 public class YoutubeCommentsExtractor extends CommentsExtractor {
+    private static final String TAG = YoutubeCommentsExtractor.class.getSimpleName();
 
     private static final String COMMENT_VIEW_MODEL_KEY = "commentViewModel";
     private static final String COMMENT_RENDERER_KEY = "commentRenderer";
@@ -43,6 +44,16 @@
      */
     private JsonObject ajaxJson;
 
+    /**
+     * Live chat continuation token, used when regular comments are disabled.
+     */
+    private String liveChatContinuation;
+
+    /**
+     * Whether this video is / was a live stream.
+     */
+    private boolean isLiveStream;
+
     public YoutubeCommentsExtractor(
             final StreamingService service,
             final ListLinkHandler uiHandler) {
@@ -54,6 +65,10 @@
     public InfoItemsPage<CommentsInfoItem> getInitialPage()
             throws IOException, ExtractionException {
 
+        if (commentsDisabled && liveChatContinuation != null) {
+            return fetchLiveChat(liveChatContinuation);
+        }
+
         if (commentsDisabled) {
             return getInfoItemsPageForDisabledComments();
         }
@@ -194,6 +209,12 @@
     public InfoItemsPage<CommentsInfoItem> getPage(final Page page)
             throws IOException, ExtractionException {
 
+        if ("live_chat".equals(page.getUrl())
+                || (commentsDisabled && liveChatContinuation != null)) {
+            isLiveStream = true;
+            return fetchLiveChat(page.getId());
+        }
+
         if (commentsDisabled) {
             return getInfoItemsPageForDisabledComments();
         }
@@ -206,7 +227,7 @@
         // @formatter:off
         final byte[] body = JsonWriter.string(
                 prepareDesktopJsonBuilder(localization, getExtractorContentCountry())
                     .value("continuation", page.getId())
                     .done())
                 .getBytes(StandardCharsets.UTF_8);
         // @formatter:on
@@ -351,10 +372,12 @@
                 .getBytes(StandardCharsets.UTF_8);
         // @formatter:on
 
-        final String initialToken =
-                findInitialCommentsToken(getJsonPostResponse("next", body, localization));
+        final JsonObject nextResponse = getJsonPostResponse("next", body, localization);
+        final String initialToken = findInitialCommentsToken(nextResponse);
 
         if (initialToken == null) {
+            // Try to extract live chat continuation for live streams
+            findLiveChatContinuation(nextResponse);
             return;
         }
 
@@ -369,10 +392,123 @@
         ajaxJson = getJsonPostResponse("next", ajaxBody, localization);
     }
 
+    /**
+     * Tries to extract a live chat continuation token from the next response.
+     * This is used when regular comments are disabled on a live stream.
+     */
+    private void findLiveChatContinuation(final JsonObject nextResponse) {
+        try {
+            final JsonObject liveChatRenderer = nextResponse
+                    .getObject("contents")
+                    .getObject("twoColumnWatchNextResults")
+                    .getObject("conversationBar")
+                    .getObject("liveChatRenderer");
+            liveChatContinuation = liveChatRenderer
+                    .getArray("continuations")
+                    .getObject(0)
+                    .getObject("reloadContinuationData")
+                    .getString("continuation");
+        } catch (final Exception e) {
+            liveChatContinuation = null;
+        }
+    }
+
+    /**
+     * Fetches live chat messages and converts them to CommentsInfoItem.
+     */
+    private InfoItemsPage<CommentsInfoItem> fetchLiveChat(final String chatContinuation)
+            throws IOException, ExtractionException {
+        isLiveStream = true;
+        final Localization localization = getExtractorLocalization();
+        final byte[] json = JsonWriter.string(
+                prepareDesktopJsonBuilder(localization, getExtractorContentCountry())
+                        .value("continuation", chatContinuation)
+                        .object("currentPlayerState")
+                        .value("playerOffsetMs", "0")
+                        .end()
+                        .done())
+                .getBytes(StandardCharsets.UTF_8);
+
+        final String endpoint = "live_chat/"
+                + (isLiveStream ? "get_live_chat" : "get_live_chat_replay");
+        final JsonObject result = getJsonPostResponse(endpoint, json, localization);
+
+        return extractLiveChatComments(result);
+    }
+
+    /**
+     * Extracts live chat actions into CommentsInfoItem objects.
+     */
+    private InfoItemsPage<CommentsInfoItem> extractLiveChatComments(
+            final JsonObject result) throws ExtractionException {
+        final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(
+                getServiceId());
+
+        try {
+            final JsonObject chatContinuation = result
+                    .getObject("continuationContents")
+                    .getObject("liveChatContinuation");
+            final JsonArray actions = chatContinuation.getArray("actions");
+
+            for (int i = 0; i < actions.size(); i++) {
+                final JsonObject action = actions.getObject(i);
+                final JsonObject item;
+                if (action.has("addChatItemAction")) {
+                    item = action.getObject("addChatItemAction")
+                            .getObject("item");
+                } else if (action.has("replayChatItemAction")) {
+                    item = action.getObject("replayChatItemAction")
+                            .getArray("actions").getObject(0)
+                            .getObject("addChatItemAction")
+                            .getObject("item");
+                } else {
+                    continue;
+                }
+
+                if (item.has("liveChatTextMessageRenderer")) {
+                    collector.commit(new YoutubeLiveChatInfoItemExtractor(
+                            item.getObject("liveChatTextMessageRenderer")));
+                }
+            }
+
+            // Extract next continuation
+            final JsonArray continuations = chatContinuation
+                    .getArray("continuations");
+            final Page nextPage;
+            if (!continuations.isEmpty()) {
+                final JsonObject contObj = continuations.getObject(
+                        continuations.size() - 1);
+                String nextCont = null;
+                if (contObj.has("timedContinuationData")) {
+                    nextCont = contObj.getObject("timedContinuationData")
+                            .getString("continuation");
+                } else if (contObj.has("invalidationContinuationData")) {
+                    nextCont = contObj.getObject("invalidationContinuationData")
+                            .getString("continuation");
+                } else if (contObj.has("liveChatReplayContinuationData")) {
+                    nextCont = contObj.getObject("liveChatReplayContinuationData")
+                            .getString("continuation");
+                }
+                nextPage = nextCont != null ? new Page("live_chat", nextCont) : null;
+            } else {
+                nextPage = null;
+            }
+
+            return new InfoItemsPage<>(collector, nextPage);
+        } catch (final Exception e) {
+            return getInfoItemsPageForDisabledComments();
+        }
+    }
+
 
     @Override
     public boolean isCommentsDisabled() {
-        return commentsDisabled;
+        return commentsDisabled && !isLiveChat();
+    }
+
+    @Override
+    public boolean isLiveChat() {
+        return liveChatContinuation != null;
     }
 
     @Override