From c1e0cb07650616dd6c7d0ed32dfc032d8388fb66 Mon Sep 17 00:00:00 2001 From: David Asunmo <22662897+davidasunmo@users.noreply.github.com.> Date: Mon, 26 May 2025 04:39:10 +0100 Subject: [PATCH 1/9] [SoundCloud] Remove redundant override --- .../services/soundcloud/SoundcloudStreamExtractorTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorTest.java index a4e81dd9c8..ec851faf24 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorTest.java @@ -133,7 +133,6 @@ public void testRelatedItems() throws Exception { @Override public long expectedDislikeCountAtLeast() { return -1; } @Override public boolean expectedHasAudioStreams() { return false; } @Override public boolean expectedHasVideoStreams() { return false; } - @Override public boolean expectedHasRelatedItems() { return true; } @Override public boolean expectedHasSubtitles() { return false; } @Override public boolean expectedHasFrames() { return false; } @Override public int expectedStreamSegmentsCount() { return 0; } From 76dc750e80606b858e0e8b6db00ccb5dec6ebe76 Mon Sep 17 00:00:00 2001 From: David Asunmo <22662897+davidasunmo@users.noreply.github.com.> Date: Wed, 4 Jun 2025 05:02:53 +0100 Subject: [PATCH 2/9] [SoundCloud] annotate inner classes to remove warning/error of nested class tests not being executed --- .../SoundcloudStreamExtractorTest.java | 39 ++++++++++++++----- 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorTest.java index ec851faf24..db68fd518d 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorTest.java @@ -2,7 +2,9 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; import org.schabi.newpipe.downloader.DownloaderTestImpl; import org.schabi.newpipe.extractor.ExtractorAsserts; import org.schabi.newpipe.extractor.MediaFormat; @@ -30,15 +32,20 @@ public class SoundcloudStreamExtractorTest { private static final String SOUNDCLOUD = "https://soundcloud.com/"; - public static class SoundcloudGeoRestrictedTrack extends DefaultStreamExtractorTest { + @Nested + @TestInstance(TestInstance.Lifecycle.PER_CLASS) + class SoundcloudGeoRestrictedTrack extends DefaultStreamExtractorTest { private static final String ID = "one-touch"; private static final String UPLOADER = SOUNDCLOUD + "jessglynne"; private static final int TIMESTAMP = 0; private static final String URL = UPLOADER + "/" + ID + "#t=" + TIMESTAMP; - private static StreamExtractor extractor; + private StreamExtractor extractor; @BeforeAll - public static void setUp() throws Exception { + public void setUp() throws Exception { + if (extractor != null) { + throw new IllegalStateException("extractor already initialized before BeforeAll"); + } NewPipe.init(DownloaderTestImpl.getInstance()); extractor = SoundCloud.getStreamExtractor(URL); try { @@ -84,15 +91,20 @@ public void testRelatedItems() throws Exception { } } - public static class SoundcloudGoPlusTrack extends DefaultStreamExtractorTest { + @Nested + @TestInstance(TestInstance.Lifecycle.PER_CLASS) + class SoundcloudGoPlusTrack extends DefaultStreamExtractorTest { private static final String ID = "places"; private static final String UPLOADER = SOUNDCLOUD + "martinsolveig"; private static final int TIMESTAMP = 0; private static final String URL = UPLOADER + "/" + ID + "#t=" + TIMESTAMP; - private static StreamExtractor extractor; + private StreamExtractor extractor; @BeforeAll - public static void setUp() throws Exception { + public void setUp() throws Exception { + if (extractor != null) { + throw new IllegalStateException("extractor already initialized before BeforeAll"); + } NewPipe.init(DownloaderTestImpl.getInstance()); extractor = SoundCloud.getStreamExtractor(URL); try { @@ -140,15 +152,20 @@ public void testRelatedItems() throws Exception { @Override public String expectedCategory() { return "Dance"; } } - static class CreativeCommonsOpenMindsEp21 extends DefaultStreamExtractorTest { + @Nested + @TestInstance(TestInstance.Lifecycle.PER_CLASS) + class CreativeCommonsOpenMindsEp21 extends DefaultStreamExtractorTest { private static final String ID = "open-minds-ep-21-dr-beth-harris-and-dr-steven-zucker-of-smarthistory"; private static final String UPLOADER = SOUNDCLOUD + "wearecc"; private static final int TIMESTAMP = 69; private static final String URL = UPLOADER + "/" + ID + "#t=" + TIMESTAMP; - private static StreamExtractor extractor; + private StreamExtractor extractor; @BeforeAll - static void setUp() throws Exception { + public void setUp() throws Exception { + if (extractor != null) { + throw new IllegalStateException("extractor already initialized before BeforeAll"); + } NewPipe.init(DownloaderTestImpl.getInstance()); extractor = SoundCloud.getStreamExtractor(URL); extractor.fetchPage(); @@ -181,6 +198,10 @@ static void setUp() throws Exception { @Override public boolean expectedHasFrames() { return false; } @Override public int expectedStreamSegmentsCount() { return 0; } @Override public String expectedLicence() { return "cc-by"; } + @Override public String expectedCategory() { return "Podcast"; } + @Override public List expectedTags() { + return Arrays.asList("ants", "collaboration", "creative commons", "stigmergy", "storytelling", "wikipedia"); + } @Override @Test From 811d52519cd14de2e6acd329ca71ff0ffb9bb43e Mon Sep 17 00:00:00 2001 From: David Asunmo <22662897+davidasunmo@users.noreply.github.com.> Date: Wed, 4 Jun 2025 05:06:13 +0100 Subject: [PATCH 3/9] [SoundCloud] SoundCloudStreamExtractor.getTimeStamp return 0 if no timestamp in url --- .../soundcloud/extractors/SoundcloudStreamExtractor.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java index 67cd533e9c..595862bde7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java @@ -121,7 +121,8 @@ public long getLength() { @Override public long getTimeStamp() throws ParsingException { - return getTimestampSeconds("(#t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)"); + final var timestamp = getTimestampSeconds("(#t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)"); + return timestamp == -2 ? 0 : timestamp; } @Override @@ -170,7 +171,7 @@ public List getAudioStreams() throws ExtractionException { try { final JsonArray transcodings = track.getObject("media") - .getArray("transcodings"); + .getArray("transcodings"); if (!isNullOrEmpty(transcodings)) { // Get information about what stream formats are available extractAudioStreams(transcodings, audioStreams); From 0199af94c12a8c614107432bfd62885c978648b0 Mon Sep 17 00:00:00 2001 From: David Asunmo <22662897+davidasunmo@users.noreply.github.com.> Date: Sat, 7 Jun 2025 08:22:55 +0100 Subject: [PATCH 4/9] [Localization] Add toString to DateWrapper for better debugging --- .../newpipe/extractor/localization/DateWrapper.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/localization/DateWrapper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/localization/DateWrapper.java index ffc29a61ce..2f0a2bdd2a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/localization/DateWrapper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/localization/DateWrapper.java @@ -69,4 +69,12 @@ public OffsetDateTime offsetDateTime() { public boolean isApproximation() { return isApproximation; } + + @Override + public String toString() { + return "DateWrapper{" + + "offsetDateTime=" + offsetDateTime + + ", isApproximation=" + isApproximation + + '}'; + } } From 91b07d129df3fde1246be2a05bfdc5ef1f86099c Mon Sep 17 00:00:00 2001 From: David Asunmo <22662897+davidasunmo@users.noreply.github.com.> Date: Sun, 8 Jun 2025 11:49:10 +0100 Subject: [PATCH 5/9] [SoundCloud] Use Pattern instead of string for regex in SoundcloudStreamLinkHandlerFactory Use non-capturing groups in regex Refactor Parser.java Add more utility methods (will be used in later commits) --- .../SoundcloudStreamLinkHandlerFactory.java | 19 +++- .../newpipe/extractor/utils/Parser.java | 105 +++++++++++++----- .../schabi/newpipe/extractor/utils/Utils.java | 10 ++ 3 files changed, 102 insertions(+), 32 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudStreamLinkHandlerFactory.java index e7809c52a1..421022ef0a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudStreamLinkHandlerFactory.java @@ -1,5 +1,7 @@ package org.schabi.newpipe.extractor.services.soundcloud.linkHandler; +import java.util.regex.Pattern; + import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper; @@ -9,11 +11,18 @@ public final class SoundcloudStreamLinkHandlerFactory extends LinkHandlerFactory { private static final SoundcloudStreamLinkHandlerFactory INSTANCE = new SoundcloudStreamLinkHandlerFactory(); - private static final String URL_PATTERN = "^https?://(www\\.|m\\.|on\\.)?" - + "soundcloud.com/[0-9a-z_-]+" - + "/(?!(tracks|albums|sets|reposts|followers|following)/?$)[0-9a-z_-]+/?([#?].*)?$"; - private static final String API_URL_PATTERN = "^https?://api-v2\\.soundcloud.com" - + "/(tracks|albums|sets|reposts|followers|following)/([0-9a-z_-]+)/"; + + private static final Pattern URL_PATTERN = Pattern.compile( + "^https?://(?:www\\.|m\\.|on\\.)?" + + "soundcloud.com/[0-9a-z_-]+" + + "/(?!(?:tracks|albums|sets|reposts|followers|following)/?$)[0-9a-z_-]+/?(?:[#?].*)?$" + ); + + private static final Pattern API_URL_PATTERN = Pattern.compile( + "^https?://api-v2\\.soundcloud.com" + + "/(tracks|albums|sets|reposts|followers|following)/([0-9a-z_-]+)/" + ); + private SoundcloudStreamLinkHandlerFactory() { } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java index cb28c5e6f7..3137585633 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java @@ -44,38 +44,83 @@ public RegexException(final String message) { } } + @Nonnull + public static Matcher matchOrThrow(@Nonnull final Pattern pattern, + final String input) throws RegexException { + final Matcher matcher = pattern.matcher(input); + if (matcher.find()) { + return matcher; + } else { + String errorMessage = "Failed to find pattern \"" + pattern.pattern() + "\""; + if (input.length() <= 1024) { + errorMessage += " inside of \"" + input + "\""; + } + throw new RegexException(errorMessage); + } + } + + @Nonnull + public static String matchNamedGroup(final String pattern, + final String input, + final String groupName) throws RegexException { + return matchNamedGroup(Pattern.compile(pattern), input, groupName); + } + + @Nonnull + public static String matchNamedGroup(@Nonnull final Pattern pattern, + final String input, + @Nonnull final String groupName) throws RegexException { + return matchOrThrow(pattern, input).group(groupName); + } + + public static int getStartIndexOfNamedGroup(@Nonnull final Pattern pattern, + final String input, + @Nonnull final String groupName) + throws RegexException { + return matchOrThrow(pattern, input).start(groupName); + } + + public static int getEndIndexOfNamedGroup(@Nonnull final Pattern pattern, + final String input, + @Nonnull final String groupName) + throws RegexException { + return matchOrThrow(pattern, input).end(groupName); + } + + @Nonnull public static String matchGroup1(final String pattern, final String input) throws RegexException { return matchGroup(pattern, input, 1); } + + @Nonnull public static String matchGroup1(final Pattern pattern, - final String input) throws RegexException { + final String input) throws RegexException { return matchGroup(pattern, input, 1); } - + + /** + * Matches the specified group of the given pattern against the input. + * + * @param pattern The regex pattern to match. + * @param input The input string to match against. + * @param group The group number to retrieve (1-based index). + * @return The matching group as a string. + * @throws RegexException If the pattern does not match the input or if the group is not found. + */ + @Nonnull public static String matchGroup(final String pattern, - final String input, - final int group) throws RegexException { + final String input, + final int group) throws RegexException { return matchGroup(Pattern.compile(pattern), input, group); } - - public static String matchGroup(@Nonnull final Pattern pat, + + @Nonnull + public static String matchGroup(@Nonnull final Pattern pattern, final String input, final int group) throws RegexException { - final Matcher matcher = pat.matcher(input); - final boolean foundMatch = matcher.find(); - if (foundMatch) { - return matcher.group(group); - } else { - // only pass input to exception message when it is not too long - if (input.length() > 1024) { - throw new RegexException("Failed to find pattern \"" + pat.pattern() + "\""); - } else { - throw new RegexException("Failed to find pattern \"" + pat.pattern() - + "\" inside of \"" + input + "\""); - } - } + return matchOrThrow(pattern, input).group(group); } public static String matchGroup1MultiplePatterns(final Pattern[] patterns, final String input) @@ -83,11 +128,20 @@ public static String matchGroup1MultiplePatterns(final Pattern[] patterns, final return matchMultiplePatterns(patterns, input).group(1); } + /** + * Matches multiple patterns against the input string and + * returns the first successful matcher + * + * @param patterns The array of regex patterns to match. + * @param input The input string to match against. + * @return A {@code Matcher} for the first successful match. + * @throws RegexException If no patterns match the input or if {@code patterns} is empty. + */ public static Matcher matchMultiplePatterns(final Pattern[] patterns, final String input) throws RegexException { - Parser.RegexException exception = null; - for (final Pattern pattern : patterns) { - final Matcher matcher = pattern.matcher(input); + RegexException exception = null; + for (final var pattern : patterns) { + final var matcher = pattern.matcher(input); if (matcher.find()) { return matcher; } else if (exception == null) { @@ -110,14 +164,11 @@ public static Matcher matchMultiplePatterns(final Pattern[] patterns, final Stri } public static boolean isMatch(final String pattern, final String input) { - final Pattern pat = Pattern.compile(pattern); - final Matcher mat = pat.matcher(input); - return mat.find(); + return isMatch(Pattern.compile(pattern), input); } public static boolean isMatch(@Nonnull final Pattern pattern, final String input) { - final Matcher mat = pattern.matcher(input); - return mat.find(); + return pattern.matcher(input).find(); } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java index c061ce30fa..38eb5b33f2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java @@ -110,6 +110,16 @@ public static long mixedNumberWordToLong(final String numberWord) * @param url the url to be tested */ public static void checkUrl(final String pattern, final String url) throws ParsingException { + checkUrl(Pattern.compile(pattern), url); + } + + /** + * Check if the url matches the pattern. + * + * @param pattern the pattern that will be used to check the url + * @param url the url to be tested + */ + public static void checkUrl(final Pattern pattern, final String url) throws ParsingException { if (isNullOrEmpty(url)) { throw new IllegalArgumentException("Url can't be null or empty"); } From 2a10a4a5c3081d7cb0563efe306a6544c1442c6f Mon Sep 17 00:00:00 2001 From: David Asunmo <22662897+davidasunmo@users.noreply.github.com.> Date: Tue, 8 Jul 2025 08:13:22 +0100 Subject: [PATCH 6/9] [SoundCloud] Refactor SoundcloudStreamExtractorTest to define test from Immutables test case object Add Immutables annotation processor to gradle build Add custom Immutables Style Add ISoundcloudStreamExtractorTestCase for use in stream extractor tests Refactor testAudioStreams to match mp3 cdn url by regex --- build.gradle | 1 + extractor/build.gradle | 6 + .../newpipe/extractor/utils/Parser.java | 6 +- .../newpipe/extractor/ExtractorAsserts.java | 8 + .../newpipe/extractor/ImmutableStyle.java | 30 ++++ ...dDefaultSoundcloudStreamExtractorTest.java | 57 +++++++ ...rameterisedDefaultStreamExtractorTest.java | 88 ++++++++++ .../SoundcloudStreamExtractorTest.java | 157 ++++++++---------- .../testcases/DefaultExtractorTestCase.java | 17 ++ .../DefaultStreamExtractorTestCase.java | 90 ++++++++++ .../ISoundcloudStreamExtractorTestCase.java | 113 +++++++++++++ 11 files changed, 484 insertions(+), 89 deletions(-) create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/ImmutableStyle.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/ParameterisedDefaultSoundcloudStreamExtractorTest.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/ParameterisedDefaultStreamExtractorTest.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/testcases/DefaultExtractorTestCase.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/testcases/DefaultStreamExtractorTestCase.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/services/testcases/ISoundcloudStreamExtractorTestCase.java diff --git a/build.gradle b/build.gradle index 247a08c167..ec7b981a0d 100644 --- a/build.gradle +++ b/build.gradle @@ -31,6 +31,7 @@ allprojects { jsr305Version = "3.0.2" junitVersion = "5.13.3" checkstyleVersion = "10.4" + immutablesVersion = "2.10.1" } } diff --git a/extractor/build.gradle b/extractor/build.gradle index 6bb86818e5..9d08ee5b21 100644 --- a/extractor/build.gradle +++ b/extractor/build.gradle @@ -46,4 +46,10 @@ dependencies { testImplementation "com.squareup.okhttp3:okhttp:4.12.0" testImplementation 'com.google.code.gson:gson:2.13.1' + testImplementation "org.immutables:value:$immutablesVersion" + testAnnotationProcessor "org.immutables:value:$immutablesVersion" + +} +repositories { + mavenCentral() } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java index 3137585633..6462f2e51f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java @@ -99,7 +99,7 @@ public static String matchGroup1(final Pattern pattern, final String input) throws RegexException { return matchGroup(pattern, input, 1); } - + /** * Matches the specified group of the given pattern against the input. * @@ -115,7 +115,7 @@ public static String matchGroup(final String pattern, final int group) throws RegexException { return matchGroup(Pattern.compile(pattern), input, group); } - + @Nonnull public static String matchGroup(@Nonnull final Pattern pattern, final String input, @@ -129,7 +129,7 @@ public static String matchGroup1MultiplePatterns(final Pattern[] patterns, final } /** - * Matches multiple patterns against the input string and + * Matches multiple patterns against the input string and * returns the first successful matcher * * @param patterns The array of regex patterns to match. diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/ExtractorAsserts.java b/extractor/src/test/java/org/schabi/newpipe/extractor/ExtractorAsserts.java index e5bed1d69d..7b2f06c436 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/ExtractorAsserts.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/ExtractorAsserts.java @@ -8,6 +8,7 @@ import java.util.Collections; import java.util.List; import java.util.Set; +import java.util.regex.Pattern; import java.util.stream.Collectors; import javax.annotation.Nonnull; @@ -168,6 +169,13 @@ public static void assertContains( "'" + shouldBeContained + "' should be contained inside '" + container + "'"); } + public static void assertMatches(final Pattern pattern, final String input) { + assertNotNull(pattern, "pattern is null"); + assertNotNull(input, "input is null"); + assertTrue(pattern.matcher(input).find(), + "Pattern '" + pattern + "' not found in input '" + input + "'"); + } + public static void assertTabsContain(@Nonnull final List tabs, @Nonnull final String... expectedTabs) { final Set tabSet = tabs.stream() diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/ImmutableStyle.java b/extractor/src/test/java/org/schabi/newpipe/extractor/ImmutableStyle.java new file mode 100644 index 0000000000..c69ea5aff8 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/ImmutableStyle.java @@ -0,0 +1,30 @@ +package org.schabi.newpipe.extractor; + +import org.immutables.value.Value; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Target; + +// CHECKSTYLE:OFF +/** + * Custom style for generated Immutables. + * See Style. + *

+ * - Abstract types start with 'I' (e.g., IExample).

+ * - Concrete immutable types do not have a prefix (e.g., Example).

+ * - Getters are prefixed with 'get', 'is', or no prefix.

+ * - Strict builder pattern is enforced.

+ */ +// CHECKSTYLE:ON +@Target({ElementType.PACKAGE, ElementType.TYPE}) +@Value.Style( + get = {"get*", "is*", "*"}, // Methods matching these prefixes will be used as getters. + // Methods matching these patterns can NOT be used as setters. + typeAbstract = {"I*"}, // Abstract types start with I + typeImmutable = "*", // Generated concrete Immutable types will not have the I prefix + visibility = Value.Style.ImplementationVisibility.PUBLIC, + strictBuilder = true, + defaultAsDefault = true, // https://immutables.github.io/immutable.html#default-attributes + jdkOnly = true +) +public @interface ImmutableStyle { } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/ParameterisedDefaultSoundcloudStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/ParameterisedDefaultSoundcloudStreamExtractorTest.java new file mode 100644 index 0000000000..5cdf7d9330 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/ParameterisedDefaultSoundcloudStreamExtractorTest.java @@ -0,0 +1,57 @@ +package org.schabi.newpipe.extractor.services; + +import org.junit.jupiter.api.Test; +import org.schabi.newpipe.extractor.ExtractorAsserts; +import org.schabi.newpipe.extractor.MediaFormat; +import org.schabi.newpipe.extractor.services.testcases.SoundcloudStreamExtractorTestCase; +import org.schabi.newpipe.extractor.stream.AudioStream; +import org.schabi.newpipe.extractor.stream.DeliveryMethod; + +import java.util.List; +import java.util.regex.Pattern; + +import static org.junit.jupiter.api.Assertions.*; + +public abstract class ParameterisedDefaultSoundcloudStreamExtractorTest + extends ParameterisedDefaultStreamExtractorTest { + protected ParameterisedDefaultSoundcloudStreamExtractorTest(SoundcloudStreamExtractorTestCase testCase) { + super(testCase); + } + + final Pattern mp3CdnUrlPattern = Pattern.compile("-media\\.sndcdn\\.com/[a-zA-Z0-9]{12}\\.128\\.mp3"); + + @Override + @Test + public void testAudioStreams() throws Exception { + super.testAudioStreams(); + final List audioStreams = extractor.getAudioStreams(); + assertEquals(3, audioStreams.size()); // 2 MP3 streams (1 progressive, 1 HLS) and 1 OPUS + audioStreams.forEach(audioStream -> { + final DeliveryMethod deliveryMethod = audioStream.getDeliveryMethod(); + final String mediaUrl = audioStream.getContent(); + if (audioStream.getFormat() == MediaFormat.OPUS) { + assertSame(DeliveryMethod.HLS, deliveryMethod, + "Wrong delivery method for stream " + audioStream.getId() + ": " + + deliveryMethod); + // Assert it's an OPUS 64 kbps media playlist URL which comes from an HLS + // SoundCloud CDN + ExtractorAsserts.assertContains("-hls-opus-media.sndcdn.com", mediaUrl); + ExtractorAsserts.assertContains(".64.opus", mediaUrl); + } else if (audioStream.getFormat() == MediaFormat.MP3) { + if (deliveryMethod == DeliveryMethod.PROGRESSIVE_HTTP) { + // Assert it's a MP3 128 kbps media URL which comes from a progressive + // SoundCloud CDN + ExtractorAsserts.assertMatches(mp3CdnUrlPattern, mediaUrl); + } else if (deliveryMethod == DeliveryMethod.HLS) { + // Assert it's a MP3 128 kbps media HLS playlist URL which comes from an HLS + // SoundCloud CDN + ExtractorAsserts.assertContains("-hls-media.sndcdn.com", mediaUrl); + ExtractorAsserts.assertContains(".128.mp3", mediaUrl); + } else { + fail("Wrong delivery method for stream " + audioStream.getId() + ": " + + deliveryMethod); + } + } + }); + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/ParameterisedDefaultStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/ParameterisedDefaultStreamExtractorTest.java new file mode 100644 index 0000000000..4a84ca6668 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/ParameterisedDefaultStreamExtractorTest.java @@ -0,0 +1,88 @@ +package org.schabi.newpipe.extractor.services; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.TestInstance; +import org.schabi.newpipe.downloader.DownloaderTestImpl; +import org.schabi.newpipe.extractor.MetaInfo; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.services.testcases.DefaultStreamExtractorTestCase; +import org.schabi.newpipe.extractor.stream.StreamExtractor; +import org.schabi.newpipe.extractor.stream.StreamType; + +import javax.annotation.Nullable; +import java.util.List; +import java.util.Locale; + +/** + * Test for {@link StreamExtractor} + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public abstract class ParameterisedDefaultStreamExtractorTest extends DefaultStreamExtractorTest { + protected TTestCase testCase; + protected StreamExtractor extractor; + + protected ParameterisedDefaultStreamExtractorTest(TTestCase testCase) + { + this.testCase = testCase; + } + + @BeforeAll + public void setUp() throws Exception { + if (extractor != null) { + throw new IllegalStateException("extractor already initialized before BeforeAll"); + } + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = testCase.service().getStreamExtractor(testCase.url()); + extractor.fetchPage(); + } + + /// + /// DefaultExtractorTest overrides + /// + + @Override public StreamExtractor extractor() throws Exception { return extractor; } + + @Override public StreamingService expectedService() throws Exception { return testCase.service(); } + @Override public String expectedName() throws Exception { return testCase.name(); } + @Override public String expectedId() throws Exception { return testCase.id(); } + @Override public String expectedUrlContains() throws Exception { return testCase.urlContains(); } + @Override public String expectedOriginalUrlContains() throws Exception { return testCase.originalUrlContains(); } + + /// + /// DefaultStreamExtractorTest overrides + /// + @Override public StreamType expectedStreamType() { return testCase.streamType(); } + @Override public String expectedUploaderName() { return testCase.uploaderName(); } + @Override public String expectedUploaderUrl() { return testCase.uploaderUrl(); } + @Override public boolean expectedUploaderVerified() { return testCase.uploaderVerified(); } + @Override public long expectedUploaderSubscriberCountAtLeast() { return testCase.uploaderSubscriberCountAtLeast(); } + @Override public String expectedSubChannelName() { return testCase.subChannelName(); } + @Override public String expectedSubChannelUrl() { return testCase.subChannelUrl(); } + @Override public boolean expectedDescriptionIsEmpty() { return testCase.descriptionIsEmpty(); } + @Override public List expectedDescriptionContains() { return testCase.descriptionContains(); } + @Override public long expectedLength() { return testCase.length(); } + @Override public long expectedTimestamp() { return testCase.timestamp(); } + @Override public long expectedViewCountAtLeast() { return testCase.viewCountAtLeast(); } + @Override @Nullable public String expectedUploadDate() { return testCase.uploadDate(); } + @Override @Nullable public String expectedTextualUploadDate() { return testCase.textualUploadDate(); } + @Override public long expectedLikeCountAtLeast() { return testCase.likeCountAtLeast(); } + @Override public long expectedDislikeCountAtLeast() { return testCase.dislikeCountAtLeast(); } + @Override public boolean expectedHasRelatedItems() { return testCase.hasRelatedItems(); } + @Override public int expectedAgeLimit() { return testCase.ageLimit(); } + @Override @Nullable public String expectedErrorMessage() { return testCase.errorMessage(); } + @Override public boolean expectedHasVideoStreams() { return testCase.hasVideoStreams(); } + @Override public boolean expectedHasAudioStreams() { return testCase.hasAudioStreams(); } + @Override public boolean expectedHasSubtitles() { return testCase.hasSubtitles(); } + @Override @Nullable public String expectedDashMpdUrlContains() { return testCase.dashMpdUrlContains(); } + @Override public boolean expectedHasFrames() { return testCase.hasFrames(); } + @Override public String expectedHost() { return testCase.host(); } + @Override public StreamExtractor.Privacy expectedPrivacy() { return testCase.privacy(); } + @Override public String expectedCategory() { return testCase.category(); } + @Override public String expectedLicence() { return testCase.licence(); } + @Override public Locale expectedLanguageInfo() { return testCase.languageInfo(); } + @Override public List expectedTags() { return testCase.tags(); } + @Override public String expectedSupportInfo() { return testCase.supportInfo(); } + @Override public int expectedStreamSegmentsCount() { return testCase.streamSegmentsCount(); } + @Override public List expectedMetaInfo() { return testCase.metaInfo(); } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorTest.java index db68fd518d..14537905b9 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorTest.java @@ -6,27 +6,21 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; import org.schabi.newpipe.downloader.DownloaderTestImpl; -import org.schabi.newpipe.extractor.ExtractorAsserts; -import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.exceptions.GeographicRestrictionException; import org.schabi.newpipe.extractor.exceptions.SoundCloudGoPlusContentException; import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest; -import org.schabi.newpipe.extractor.stream.AudioStream; -import org.schabi.newpipe.extractor.stream.DeliveryMethod; +import org.schabi.newpipe.extractor.services.ParameterisedDefaultSoundcloudStreamExtractorTest; +import org.schabi.newpipe.extractor.services.testcases.SoundcloudStreamExtractorTestCase; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.stream.StreamType; -import java.util.Arrays; import java.util.Collections; import java.util.List; import javax.annotation.Nullable; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertSame; -import static org.junit.jupiter.api.Assertions.fail; import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; public class SoundcloudStreamExtractorTest { @@ -154,89 +148,80 @@ public void testRelatedItems() throws Exception { @Nested @TestInstance(TestInstance.Lifecycle.PER_CLASS) - class CreativeCommonsOpenMindsEp21 extends DefaultStreamExtractorTest { - private static final String ID = "open-minds-ep-21-dr-beth-harris-and-dr-steven-zucker-of-smarthistory"; - private static final String UPLOADER = SOUNDCLOUD + "wearecc"; - private static final int TIMESTAMP = 69; - private static final String URL = UPLOADER + "/" + ID + "#t=" + TIMESTAMP; - private StreamExtractor extractor; + class SoundcloudTrackTest1 extends ParameterisedDefaultSoundcloudStreamExtractorTest { + public SoundcloudTrackTest1() { + super( + SoundcloudStreamExtractorTestCase.builder() + .url("https://soundcloud.com/user-904087338/nether#t=45") + .id("2057071056") + .name("Nether") + .uploaderName("Ambient Ghost") + .uploadDate("2025-03-18 12:19:19.000") + .textualUploadDate("2025-03-18 12:19:19") + .length(145) + .licence("all-rights-reserved") + .descriptionIsEmpty(true) + .viewCountAtLeast(1029) + .likeCountAtLeast(12) + .build() + ); + } + } - @BeforeAll - public void setUp() throws Exception { - if (extractor != null) { - throw new IllegalStateException("extractor already initialized before BeforeAll"); - } - NewPipe.init(DownloaderTestImpl.getInstance()); - extractor = SoundCloud.getStreamExtractor(URL); - extractor.fetchPage(); + @Nested + @TestInstance(TestInstance.Lifecycle.PER_CLASS) + class SoundcloudTrackTest2 extends ParameterisedDefaultSoundcloudStreamExtractorTest { + public SoundcloudTrackTest2() { + super( + SoundcloudStreamExtractorTestCase.builder() + .url("https://soundcloud.com/kaleidocollective/2subtact-splinter") + .id("230211123") + .name("Subtact - Splinter") + .uploaderVerified(true) + .uploaderName("Kaleido") + .uploadDate("2015-10-26 20:55:30.000") + .textualUploadDate("2015-10-26 20:55:30") + .length(225) + .licence("all-rights-reserved") + .descriptionIsEmpty(false) + .addDescriptionContains("follow @subtact", + "-twitter:", + "twitter.com/Subtact", + "-facebook:", + "www.facebook.com/subtact?fref=ts") + .viewCountAtLeast(157874) + .likeCountAtLeast(3142) + .category("ʕ•ᴥ•ʔ") + .build() + ); } - @Override public StreamExtractor extractor() { return extractor; } - @Override public StreamingService expectedService() { return SoundCloud; } - @Override public String expectedName() { return "Open Minds, Ep 21: Dr. Beth Harris and Dr. Steven Zucker of Smarthistory"; } - @Override public String expectedId() { return "1356023209"; } - @Override public String expectedUrlContains() { return UPLOADER + "/" + ID; } - @Override public String expectedOriginalUrlContains() { return URL; } + } - @Override public StreamType expectedStreamType() { return StreamType.AUDIO_STREAM; } - @Override public String expectedUploaderName() { return "Creative Commons"; } - @Override public String expectedUploaderUrl() { return UPLOADER; } - @Override public List expectedDescriptionContains() { - return Arrays.asList("Smarthistory is a center for public art history", - "experts who want to share their knowledge with learners around the world", - "Available for use under the CC BY 3.0 license"); } - @Override public long expectedLength() { return 1500; } - @Override public long expectedTimestamp() { return TIMESTAMP; } - @Override public long expectedViewCountAtLeast() { return 15000; } - @Nullable @Override public String expectedUploadDate() { return "2022-10-03 18:49:49.000"; } - @Nullable @Override public String expectedTextualUploadDate() { return "2022-10-03 18:49:49"; } - @Override public long expectedLikeCountAtLeast() { return 10; } - @Override public long expectedDislikeCountAtLeast() { return -1; } - @Override public boolean expectedHasRelatedItems() { return false; } - @Override public boolean expectedHasVideoStreams() { return false; } - @Override public boolean expectedHasSubtitles() { return false; } - @Override public boolean expectedHasFrames() { return false; } - @Override public int expectedStreamSegmentsCount() { return 0; } - @Override public String expectedLicence() { return "cc-by"; } - @Override public String expectedCategory() { return "Podcast"; } - @Override public List expectedTags() { - return Arrays.asList("ants", "collaboration", "creative commons", "stigmergy", "storytelling", "wikipedia"); + @Nested + @TestInstance(TestInstance.Lifecycle.PER_CLASS) + class SoundcloudTrackTest3 extends ParameterisedDefaultSoundcloudStreamExtractorTest { + public SoundcloudTrackTest3() { + super(SoundcloudStreamExtractorTestCase.builder() + .url("https://soundcloud.com/wearecc/open-minds-ep-21-dr-beth-harris-and-dr-steven-zucker-of-smarthistory") + .id("1356023209") + .name("Open Minds, Ep 21: Dr. Beth Harris and Dr. Steven Zucker of Smarthistory") + .uploaderName("Creative Commons") + .uploadDate("2022-10-03 18:49:49.000") + .textualUploadDate("2022-10-03 18:49:49") + .hasRelatedItems(false) + .length(1500) + .licence("cc-by") + .descriptionIsEmpty(false) + .addDescriptionContains("On this episode, we're joined by art historians", + "Follow Smarthistory on Twitter: https://twitter.com/Smarthistory", + "Open Minds … from Creative Commons is licensed to the public under CC BY", + "(https://creativecommons.org/licenses/by/4.0/)") + .viewCountAtLeast(15584) + .likeCountAtLeast(14) + .build() + ); } - @Override - @Test - public void testAudioStreams() throws Exception { - super.testAudioStreams(); - final List audioStreams = extractor.getAudioStreams(); - assertEquals(3, audioStreams.size()); // 2 MP3 streams (1 progressive, 1 HLS) and 1 OPUS - audioStreams.forEach(audioStream -> { - final DeliveryMethod deliveryMethod = audioStream.getDeliveryMethod(); - final String mediaUrl = audioStream.getContent(); - if (audioStream.getFormat() == MediaFormat.OPUS) { - assertSame(DeliveryMethod.HLS, deliveryMethod, - "Wrong delivery method for stream " + audioStream.getId() + ": " - + deliveryMethod); - // Assert it's an OPUS 64 kbps media playlist URL which comes from an HLS - // SoundCloud CDN - ExtractorAsserts.assertContains("-hls-opus-media.sndcdn.com", mediaUrl); - ExtractorAsserts.assertContains(".64.opus", mediaUrl); - } else if (audioStream.getFormat() == MediaFormat.MP3) { - if (deliveryMethod == DeliveryMethod.PROGRESSIVE_HTTP) { - // Assert it's a MP3 128 kbps media URL which comes from a progressive - // SoundCloud CDN - ExtractorAsserts.assertContains("-media.sndcdn.com/cyaz0oXJYbdt.128.mp3", - mediaUrl); - } else if (deliveryMethod == DeliveryMethod.HLS) { - // Assert it's a MP3 128 kbps media HLS playlist URL which comes from an HLS - // SoundCloud CDN - ExtractorAsserts.assertContains("-hls-media.sndcdn.com", mediaUrl); - ExtractorAsserts.assertContains(".128.mp3", mediaUrl); - } else { - fail("Wrong delivery method for stream " + audioStream.getId() + ": " - + deliveryMethod); - } - } - }); - } } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/testcases/DefaultExtractorTestCase.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/testcases/DefaultExtractorTestCase.java new file mode 100644 index 0000000000..f43656161d --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/testcases/DefaultExtractorTestCase.java @@ -0,0 +1,17 @@ +package org.schabi.newpipe.extractor.services.testcases; + +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.services.DefaultExtractorTest; + +/** + * Test case base class for {@link DefaultExtractorTest} + */ +public interface DefaultExtractorTestCase { + public abstract StreamingService service(); + public abstract String name(); + public abstract String id(); + public abstract String url(); + public default String originalUrl() { return url(); } + public default String urlContains() { return url();} + public default String originalUrlContains() { return originalUrl(); } +} \ No newline at end of file diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/testcases/DefaultStreamExtractorTestCase.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/testcases/DefaultStreamExtractorTestCase.java new file mode 100644 index 0000000000..4190b5cebd --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/testcases/DefaultStreamExtractorTestCase.java @@ -0,0 +1,90 @@ +package org.schabi.newpipe.extractor.services.testcases; + +import static org.schabi.newpipe.extractor.stream.StreamExtractor.UNKNOWN_SUBSCRIBER_COUNT; +import org.schabi.newpipe.extractor.MetaInfo; +import org.schabi.newpipe.extractor.stream.StreamExtractor; +import org.schabi.newpipe.extractor.stream.StreamType; + +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.regex.Matcher; + +import javax.annotation.Nullable; + +/** + * Test case base class for {@link org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest}

+ * Ideally you will supply a regex matcher that the url that will automatically parse + * certain values for the tests.

+ * Ones that can't be derived from the url should be overridden in the test case. + */ +public interface DefaultStreamExtractorTestCase extends DefaultExtractorTestCase { + /** + * Returns matcher for the URL

+ * Implementations should throw IllegalArgumentException if the pattern does not match + */ + Matcher urlMatcher(); + + default String getGroupFromUrl(String groupName) { + return urlMatcher().group(groupName); + } + + default int getGroupEndIndexFromUrl(String groupName) { + return urlMatcher().end(groupName); + } + + default String id() { return getGroupFromUrl("id"); } + + default String uploader() { return getGroupFromUrl("uploader"); } + + StreamType streamType(); + String uploaderName(); + default String uploaderUrl() { + final int groupEndIndex = getGroupEndIndexFromUrl("uploader"); + if (groupEndIndex < 0) { + return ""; // no uploader group found in url + } + return url().substring(0, groupEndIndex); + } + default boolean uploaderVerified() { return false; } + default long uploaderSubscriberCountAtLeast() { return UNKNOWN_SUBSCRIBER_COUNT; } // default: unknown + default String subChannelName() { return ""; } // default: no subchannel + default String subChannelUrl() { return ""; } // default: no subchannel + default boolean descriptionIsEmpty() { return false; } // default: description is not empty + List descriptionContains(); + long length(); + default int timestamp() { return 0; } // default: there is no timestamp + long viewCountAtLeast(); + + @Nullable + String uploadDate(); // format: yyyy-MM-dd HH:mm:ss.SSS + @Nullable + String textualUploadDate(); + long likeCountAtLeast(); + long dislikeCountAtLeast(); + default boolean hasRelatedItems() { return true; } // default: there are related videos + default int ageLimit() { return StreamExtractor.NO_AGE_LIMIT; } // default: no limit + @Nullable + default String errorMessage() { return null; } // default: no error message + default boolean hasVideoStreams() { return true; } // default: there are video streams + default boolean hasAudioStreams() { return true; } // default: there are audio streams + default boolean hasSubtitles() { return true; } // default: there are subtitles streams + @Nullable + default String dashMpdUrlContains() { return null; } // default: no dash mpd + default boolean hasFrames() { return true; } // default: there are frames + @Nullable + default String host() { return ""; } // default: no host for centralized platforms + @Nullable + default StreamExtractor.Privacy privacy() { return StreamExtractor.Privacy.PUBLIC; } // default: public + default String category() { return ""; } // default: no category + default String licence() { return ""; } // default: no licence + @Nullable + default Locale languageInfo() { return null; } // default: no language info available + @Nullable + default List tags() { return Collections.emptyList(); } // default: no tags + @Nullable + default String supportInfo() { return ""; } // default: no support info available + default int streamSegmentsCount() { return -1; } // return 0 or greater to test (default is -1 to ignore) + @Nullable + default List metaInfo() { return Collections.emptyList(); } // default: no metadata info available +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/testcases/ISoundcloudStreamExtractorTestCase.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/testcases/ISoundcloudStreamExtractorTestCase.java new file mode 100644 index 0000000000..e27a29fbc7 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/testcases/ISoundcloudStreamExtractorTestCase.java @@ -0,0 +1,113 @@ +package org.schabi.newpipe.extractor.services.testcases; + +import org.immutables.value.Value; +import org.schabi.newpipe.extractor.ImmutableStyle; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.stream.StreamType; +import org.schabi.newpipe.extractor.utils.Parser; +import org.schabi.newpipe.extractor.utils.Parser.RegexException; +import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudStreamExtractorTest; + +import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +// CHECKSTYLE:OFF +/** + * Immutable definition of {@link DefaultStreamExtractorTestCase} + * for {@link SoundcloudStreamExtractorTest} streams + * @see SoundcloudStreamExtractorTest + */ +// CHECKSTYLE:ON +@ImmutableStyle +@Value.Immutable +public interface ISoundcloudStreamExtractorTestCase extends DefaultStreamExtractorTestCase { + + /** + * Pattern for matching soundcloud stream URLs + * Matches URLs of the form: + * 

+     * https://soundcloud.com/user-904087338/nether#t=46
+     *
+ */ + Pattern URL_PATTERN = java.util.regex.Pattern.compile( + "^https?://(?:www\\.|m\\.|on\\.)?soundcloud\\.com/" + + "(?[0-9a-z_-]+)/(?!(?:tracks|albums|sets|reposts|followers|following)/?$)" + + "(?[0-9a-z_-]+)/?" + + "([#?](t=(?\\d+)|.*))?$" + ); + + /** + * Returns the named group from the URL, or an empty string if not found. + */ + default String getGroupFromUrl(String group) { + try { + final String value = urlMatcher().group(group); + return value != null ? value : ""; + } catch (IllegalArgumentException | IllegalStateException e) { + return ""; + } + } + + /** + * Returns the end index of the named group from the URL, or -1 if not found. + */ + default int getGroupEndIndexFromUrl(String group) { + try { + return urlMatcher().end(group); + } catch (IllegalArgumentException | IllegalStateException e) { + return -1; + } + } + + /** + * @inheritdoc + */ + default Matcher urlMatcher() { + try { + return Parser.matchOrThrow(URL_PATTERN, url()); + } catch (RegexException e) { + throw new IllegalArgumentException("URL does not match expected SoundCloud pattern: " + url(), e); + } + } + + default String urlContains() { + final int groupEndIndex = getGroupEndIndexFromUrl("id"); + if (groupEndIndex < 0) { + return url(); // no id group found in url + } + return url().substring(0, groupEndIndex); + } + + @Value.Derived + public default StreamingService service() { return SoundCloud; } + + @Value.Derived + @Override + public default StreamType streamType() { return StreamType.AUDIO_STREAM; } + + @Override + public default int timestamp() { + try { + return Integer.parseInt(getGroupFromUrl("timestamp")); + } + catch (NumberFormatException e) { + // Return 0 if no timestamp + return 0; + } + } + + @Override + public default long dislikeCountAtLeast() { return -1; } // default: soundcloud has no dislikes + + @Override + public default boolean hasVideoStreams() { return false; } // default: soundcloud has no video streams + + @Override + public default boolean hasSubtitles() { return false; } // default: soundcloud has no subtitles + + public default boolean hasFrames() { return false; } // default: soundcloud has no frames + + public default int streamSegmentsCount() { return 0; } +} From 212886e64245a6eccceea447b34376ec78ec3759 Mon Sep 17 00:00:00 2001 From: David Asunmo <22662897+davidasunmo@users.noreply.github.com.> Date: Tue, 8 Jul 2025 07:47:04 +0100 Subject: [PATCH 7/9] Revert named group regex refactoring in Parser.java (does not work on API <26) Add javadocs to other methods in Parser.java --- .../newpipe/extractor/utils/Parser.java | 81 ++++++++++--------- 1 file changed, 45 insertions(+), 36 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java index 6462f2e51f..0cf4654e6a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java @@ -59,49 +59,39 @@ public static Matcher matchOrThrow(@Nonnull final Pattern pattern, } } - @Nonnull - public static String matchNamedGroup(final String pattern, - final String input, - final String groupName) throws RegexException { - return matchNamedGroup(Pattern.compile(pattern), input, groupName); - } - - @Nonnull - public static String matchNamedGroup(@Nonnull final Pattern pattern, - final String input, - @Nonnull final String groupName) throws RegexException { - return matchOrThrow(pattern, input).group(groupName); - } - - public static int getStartIndexOfNamedGroup(@Nonnull final Pattern pattern, - final String input, - @Nonnull final String groupName) - throws RegexException { - return matchOrThrow(pattern, input).start(groupName); - } - - public static int getEndIndexOfNamedGroup(@Nonnull final Pattern pattern, - final String input, - @Nonnull final String groupName) - throws RegexException { - return matchOrThrow(pattern, input).end(groupName); - } - + /** + * Matches group 1 of the given pattern against the input + * and returns the matched group + * + * @param pattern The regex pattern to match. + * @param input The input string to match against. + * @return The matching group as a string. + * @throws RegexException If the pattern does not match the input or if the group is not found. + */ @Nonnull public static String matchGroup1(final String pattern, final String input) throws RegexException { return matchGroup(pattern, input, 1); } - + /** + * Matches group 1 of the given pattern against the input + * and returns the matched group + * + * @param pattern The regex pattern to match. + * @param input The input string to match against. + * @return The matching group as a string. + * @throws RegexException If the pattern does not match the input or if the group is not found. + */ @Nonnull - public static String matchGroup1(final Pattern pattern, - final String input) throws RegexException { + public static String matchGroup1(final Pattern pattern, final String input) + throws RegexException { return matchGroup(pattern, input, 1); } /** - * Matches the specified group of the given pattern against the input. + * Matches the specified group of the given pattern against the input, + * and returns the matched group * * @param pattern The regex pattern to match. * @param input The input string to match against. @@ -110,19 +100,38 @@ public static String matchGroup1(final Pattern pattern, * @throws RegexException If the pattern does not match the input or if the group is not found. */ @Nonnull - public static String matchGroup(final String pattern, - final String input, - final int group) throws RegexException { + public static String matchGroup(final String pattern, final String input, final int group) + throws RegexException { return matchGroup(Pattern.compile(pattern), input, group); } + /** + * Matches the specified group of the given pattern against the input, + * and returns the matched group + * + * @param pattern The regex pattern to match. + * @param input The input string to match against. + * @param group The group number to retrieve (1-based index). + * @return The matching group as a string. + * @throws RegexException If the pattern does not match the input or if the group is not found. + */ @Nonnull public static String matchGroup(@Nonnull final Pattern pattern, final String input, - final int group) throws RegexException { + final int group) + throws RegexException { return matchOrThrow(pattern, input).group(group); } + /** + * Matches multiple patterns against the input string and + * returns the first successful matcher + * + * @param patterns The array of regex patterns to match. + * @param input The input string to match against. + * @return A {@code Matcher} for the first successful match. + * @throws RegexException If no patterns match the input or if {@code patterns} is empty. + */ public static String matchGroup1MultiplePatterns(final Pattern[] patterns, final String input) throws RegexException { return matchMultiplePatterns(patterns, input).group(1); From 0b086ce44d0a565d147a542173c5505e374d8df6 Mon Sep 17 00:00:00 2001 From: David Asunmo <22662897+davidasunmo@users.noreply.github.com.> Date: Thu, 3 Jul 2025 05:26:01 +0100 Subject: [PATCH 8/9] [SoundCloud] Minor refactors --- .../SoundcloudStreamExtractorTest.java | 2 -- .../ISoundcloudStreamExtractorTestCase.java | 23 ++++++++----------- 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorTest.java index 14537905b9..5d9998091e 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorTest.java @@ -195,7 +195,6 @@ public SoundcloudTrackTest2() { .build() ); } - } @Nested @@ -222,6 +221,5 @@ public SoundcloudTrackTest3() { .build() ); } - } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/testcases/ISoundcloudStreamExtractorTestCase.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/testcases/ISoundcloudStreamExtractorTestCase.java index e27a29fbc7..7a477abfad 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/testcases/ISoundcloudStreamExtractorTestCase.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/testcases/ISoundcloudStreamExtractorTestCase.java @@ -28,10 +28,10 @@ public interface ISoundcloudStreamExtractorTestCase extends DefaultStreamExtract * Pattern for matching soundcloud stream URLs * Matches URLs of the form: * 
-     * https://soundcloud.com/user-904087338/nether#t=46
+     * https://soundcloud.com/user-904087338/nether#t=46
      *
*/ - Pattern URL_PATTERN = java.util.regex.Pattern.compile( + Pattern URL_PATTERN = Pattern.compile( "^https?://(?:www\\.|m\\.|on\\.)?soundcloud\\.com/" + "(?[0-9a-z_-]+)/(?!(?:tracks|albums|sets|reposts|followers|following)/?$)" + "(?[0-9a-z_-]+)/?" @@ -61,9 +61,6 @@ default int getGroupEndIndexFromUrl(String group) { } } - /** - * @inheritdoc - */ default Matcher urlMatcher() { try { return Parser.matchOrThrow(URL_PATTERN, url()); @@ -81,14 +78,14 @@ default String urlContains() { } @Value.Derived - public default StreamingService service() { return SoundCloud; } + default StreamingService service() { return SoundCloud; } @Value.Derived @Override - public default StreamType streamType() { return StreamType.AUDIO_STREAM; } + default StreamType streamType() { return StreamType.AUDIO_STREAM; } @Override - public default int timestamp() { + default int timestamp() { try { return Integer.parseInt(getGroupFromUrl("timestamp")); } @@ -99,15 +96,15 @@ public default int timestamp() { } @Override - public default long dislikeCountAtLeast() { return -1; } // default: soundcloud has no dislikes + default long dislikeCountAtLeast() { return -1; } // default: soundcloud has no dislikes @Override - public default boolean hasVideoStreams() { return false; } // default: soundcloud has no video streams + default boolean hasVideoStreams() { return false; } // default: soundcloud has no video streams @Override - public default boolean hasSubtitles() { return false; } // default: soundcloud has no subtitles + default boolean hasSubtitles() { return false; } // default: soundcloud has no subtitles - public default boolean hasFrames() { return false; } // default: soundcloud has no frames + default boolean hasFrames() { return false; } // default: soundcloud has no frames - public default int streamSegmentsCount() { return 0; } + default int streamSegmentsCount() { return 0; } } From b89ac78090e5f0ca3e4ee4160e1358dc567db937 Mon Sep 17 00:00:00 2001 From: David Asunmo <22662897+davidasunmo@users.noreply.github.com.> Date: Tue, 8 Jul 2025 08:00:28 +0100 Subject: [PATCH 9/9] [SoundCloud] update creative commons test properties to match dev --- .../soundcloud/SoundcloudStreamExtractorTest.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorTest.java index 5d9998091e..8405b8fd0c 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorTest.java @@ -212,12 +212,11 @@ public SoundcloudTrackTest3() { .length(1500) .licence("cc-by") .descriptionIsEmpty(false) - .addDescriptionContains("On this episode, we're joined by art historians", - "Follow Smarthistory on Twitter: https://twitter.com/Smarthistory", - "Open Minds … from Creative Commons is licensed to the public under CC BY", - "(https://creativecommons.org/licenses/by/4.0/)") - .viewCountAtLeast(15584) - .likeCountAtLeast(14) + .addDescriptionContains("Smarthistory is a center for public art history", + "experts who want to share their knowledge with learners around the world", + "Available for use under the CC BY 3.0 license") + .viewCountAtLeast(15000) + .likeCountAtLeast(10) .build() ); }