diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/Instance.java b/extractor/src/main/java/org/schabi/newpipe/extractor/Instance.java new file mode 100644 index 0000000000..da8a3784f1 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/Instance.java @@ -0,0 +1,27 @@ +package org.schabi.newpipe.extractor; + +import org.schabi.newpipe.extractor.exceptions.InvalidInstanceException; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +public interface Instance { + + @Nullable + String getName(); + + @Nonnull + String getUrl(); + + boolean isValid(); + + /** + * Fetch instance metadata. + *

+ * You can e.g. save the name + * + * @throws InvalidInstanceException + */ + void fetchInstanceMetaData() throws InvalidInstanceException; + +} \ No newline at end of file diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/NewPipe.java b/extractor/src/main/java/org/schabi/newpipe/extractor/NewPipe.java index fbb90a1899..be9290d7b3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/NewPipe.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/NewPipe.java @@ -36,6 +36,7 @@ public class NewPipe { private static Downloader downloader; private static Localization preferredLocalization; private static ContentCountry preferredContentCountry; + private static boolean useInvidiousForYouTube; private NewPipe() { } @@ -154,4 +155,13 @@ public static ContentCountry getPreferredContentCountry() { public static void setPreferredContentCountry(ContentCountry preferredContentCountry) { NewPipe.preferredContentCountry = preferredContentCountry; } + + public static boolean getUseInvidiousForYoutube() { + return useInvidiousForYouTube; + } + + public static void setUseInvidiousForYoutube(final boolean useInvidiousForYoutube) { + NewPipe.useInvidiousForYouTube = useInvidiousForYoutube; + } + } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java b/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java index 83d8522f50..69f8044cce 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java @@ -4,6 +4,7 @@ import org.schabi.newpipe.extractor.services.peertube.PeertubeService; import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudService; import org.schabi.newpipe.extractor.services.youtube.YoutubeService; +import org.schabi.newpipe.extractor.services.youtube.invidious.InvidiousService; import java.util.Arrays; import java.util.Collections; @@ -39,6 +40,7 @@ private ServiceList() { public static final SoundcloudService SoundCloud; public static final MediaCCCService MediaCCC; public static final PeertubeService PeerTube; + public static final InvidiousService Invidious; /** * When creating a new service, put this service in the end of this list, @@ -52,12 +54,20 @@ private ServiceList() { PeerTube = new PeertubeService(3) )); + private static final List SERVICES_WITH_INVIDIOUS = Collections.unmodifiableList( + Arrays.asList( + Invidious = new InvidiousService(0), + SoundCloud, + MediaCCC, + PeerTube + )); + /** * Get all the supported services. * * @return a unmodifiable list of all the supported services */ public static List all() { - return SERVICES; + return NewPipe.getUseInvidiousForYoutube() ? SERVICES_WITH_INVIDIOUS : SERVICES; } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java index dcde0aff60..ff195495cc 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingService.java @@ -52,7 +52,8 @@ public static class ServiceInfo { /** * Creates a new instance of a ServiceInfo - * @param name the name of the service + * + * @param name the name of the service * @param mediaCapabilities the type of media this service can handle */ public ServiceInfo(String name, List mediaCapabilities) { @@ -69,7 +70,7 @@ public List getMediaCapabilities() { } public enum MediaCapability { - AUDIO, VIDEO, LIVE, COMMENTS + AUDIO, VIDEO, LIVE, COMMENTS, INSTANCES } } @@ -86,15 +87,16 @@ public enum LinkType { private final int serviceId; private final ServiceInfo serviceInfo; + private Instance instance; /** * Creates a new Streaming service. * If you Implement one do not set id within your implementation of this extractor, instead - * set the id when you put the extractor into - * ServiceList. + * set the id when you put the extractor into {@link ServiceList}. * All other parameters can be set directly from the overriding constructor. - * @param id the number of the service to identify him within the NewPipe frontend - * @param name the name of the service + * + * @param id the number of the service to identify him within the NewPipe frontend + * @param name the name of the service * @param capabilities the type of media this service can handle */ public StreamingService(int id, String name, List capabilities) { @@ -123,6 +125,7 @@ public String toString() { /** * Must return a new instance of an implementation of LinkHandlerFactory for streams. + * * @return an instance of a LinkHandlerFactory for streams */ public abstract LinkHandlerFactory getStreamLHFactory(); @@ -130,6 +133,7 @@ public String toString() { /** * Must return a new instance of an implementation of ListLinkHandlerFactory for channels. * If support for channels is not given null must be returned. + * * @return an instance of a ListLinkHandlerFactory for channels or null */ public abstract ListLinkHandlerFactory getChannelLHFactory(); @@ -137,15 +141,18 @@ public String toString() { /** * Must return a new instance of an implementation of ListLinkHandlerFactory for playlists. * If support for playlists is not given null must be returned. + * * @return an instance of a ListLinkHandlerFactory for playlists or null */ public abstract ListLinkHandlerFactory getPlaylistLHFactory(); /** * Must return an instance of an implementation of SearchQueryHandlerFactory. + * * @return an instance of a SearchQueryHandlerFactory */ public abstract SearchQueryHandlerFactory getSearchQHFactory(); + public abstract ListLinkHandlerFactory getCommentsLHFactory(); /*////////////////////////////////////////////////////////////////////////// @@ -154,6 +161,7 @@ public String toString() { /** * Must create a new instance of a SearchExtractor implementation. + * * @param queryHandler specifies the keyword lock for, and the filters which should be applied. * @return a new SearchExtractor instance */ @@ -161,12 +169,14 @@ public String toString() { /** * Must create a new instance of a SuggestionExtractor implementation. + * * @return a new SuggestionExtractor instance */ public abstract SuggestionExtractor getSuggestionExtractor(); /** * Outdated or obsolete. null can be returned. + * * @return just null */ public abstract SubscriptionExtractor getSubscriptionExtractor(); @@ -186,6 +196,7 @@ public FeedExtractor getFeedExtractor(String url) throws ExtractionException { /** * Must create a new instance of a KioskList implementation. + * * @return a new KioskList instance * @throws ExtractionException */ @@ -193,6 +204,7 @@ public FeedExtractor getFeedExtractor(String url) throws ExtractionException { /** * Must create a new instance of a ChannelExtractor implementation. + * * @param linkHandler is pointing to the channel which should be handled by this new instance. * @return a new ChannelExtractor * @throws ExtractionException @@ -201,6 +213,7 @@ public FeedExtractor getFeedExtractor(String url) throws ExtractionException { /** * Must crete a new instance of a PlaylistExtractor implementation. + * * @param linkHandler is pointing to the playlist which should be handled by this new instance. * @return a new PlaylistExtractor * @throws ExtractionException @@ -209,6 +222,7 @@ public FeedExtractor getFeedExtractor(String url) throws ExtractionException { /** * Must create a new instance of a StreamExtractor implementation. + * * @param linkHandler is pointing to the stream which should be handled by this new instance. * @return a new StreamExtractor * @throws ExtractionException @@ -276,6 +290,7 @@ public CommentsExtractor getCommentsExtractor(String url) throws ExtractionExcep /** * Figures out where the link is pointing to (a channel, a video, a playlist, etc.) + * * @param url the url on which it should be decided of which link type it is * @return the link type of url */ @@ -387,4 +402,12 @@ public TimeAgoParser getTimeAgoParser(Localization localization) { throw new IllegalArgumentException("Localization is not supported (\"" + localization.toString() + "\")"); } + public Instance getInstance() { + return this.instance; + } + + public void setInstance(final Instance instance) { + this.instance = instance; + } + } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/exceptions/InvalidInstanceException.java b/extractor/src/main/java/org/schabi/newpipe/extractor/exceptions/InvalidInstanceException.java new file mode 100644 index 0000000000..84330f370a --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/exceptions/InvalidInstanceException.java @@ -0,0 +1,11 @@ +package org.schabi.newpipe.extractor.exceptions; + +public class InvalidInstanceException extends ExtractionException { + public InvalidInstanceException(String message) { + super(message); + } + + public InvalidInstanceException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ListLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ListLinkHandlerFactory.java index 86991c4df4..6dfa14672f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ListLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/ListLinkHandlerFactory.java @@ -86,7 +86,7 @@ public String getUrl(String id, String baseUrl) throws ParsingException { } /** - * Will returns content filter the corresponding extractor can handle like "channels", "videos", "music", etc. + * Will return content filter the corresponding extractor can handle like "channels", "videos", "music", etc. * * @return filter that can be applied when building a query for getting a list */ @@ -95,7 +95,7 @@ public String[] getAvailableContentFilter() { } /** - * Will returns sort filter the corresponding extractor can handle like "A-Z", "oldest first", "size", etc. + * Will return sort filter the corresponding extractor can handle like "A-Z", "oldest first", "size", etc. * * @return filter that can be applied when building a query for getting a list */ diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java index a29a592e0e..843fccb03f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java @@ -3,10 +3,11 @@ import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; - +import org.schabi.newpipe.extractor.Instance; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.downloader.Response; +import org.schabi.newpipe.extractor.exceptions.InvalidInstanceException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.utils.JsonUtils; @@ -14,15 +15,16 @@ import java.io.IOException; -public class PeertubeInstance { +public class PeertubeInstance implements Instance { private final String url; private String name; + private Boolean isValid = null; + public static final PeertubeInstance defaultInstance = new PeertubeInstance("https://framatube.org", "FramaTube"); public PeertubeInstance(String url) { - this.url = url; - this.name = "PeerTube"; + this(url, "PeerTube"); } public PeertubeInstance(String url, String name) { @@ -34,25 +36,39 @@ public String getUrl() { return url; } - public void fetchInstanceMetaData() throws Exception { + @Override + public boolean isValid() { + if (isValid != null) { + return isValid; + } + + try { + fetchInstanceMetaData(); + return isValid = true; + } catch (InvalidInstanceException e) { + return isValid = false; + } + } + + public void fetchInstanceMetaData() throws InvalidInstanceException { Downloader downloader = NewPipe.getDownloader(); - Response response = null; + Response response; try { response = downloader.get(url + "/api/v1/config"); } catch (ReCaptchaException | IOException e) { - throw new Exception("unable to configure instance " + url, e); + throw new InvalidInstanceException("unable to configure instance " + url, e); } if (response == null || Utils.isBlank(response.responseBody())) { - throw new Exception("unable to configure instance " + url); + throw new InvalidInstanceException("unable to configure instance " + url); } try { JsonObject json = JsonParser.object().from(response.responseBody()); this.name = JsonUtils.getString(json, "instance.name"); } catch (JsonParserException | ParsingException e) { - throw new Exception("unable to parse instance config", e); + throw new InvalidInstanceException("unable to parse instance config", e); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java index ea1a78442c..1fc494fd65 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeService.java @@ -18,9 +18,11 @@ import java.util.List; import static java.util.Arrays.asList; -import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.COMMENTS; -import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.VIDEO; +import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.*; +/** + * PeertubeService, uses documented API: https://docs.joinpeertube.org/api-rest-reference.html + */ public class PeertubeService extends StreamingService { private PeertubeInstance instance; @@ -29,9 +31,9 @@ public PeertubeService(int id) { this(id, PeertubeInstance.defaultInstance); } - public PeertubeService(int id, PeertubeInstance instance) { - super(id, "PeerTube", asList(VIDEO, COMMENTS)); - this.instance = instance; + public PeertubeService(int id, final PeertubeInstance instance) { + super(id, "PeerTube", asList(VIDEO, COMMENTS, INSTANCES)); + setInstance(instance); } @Override @@ -110,15 +112,7 @@ public CommentsExtractor getCommentsExtractor(ListLinkHandler linkHandler) @Override public String getBaseUrl() { - return instance.getUrl(); - } - - public PeertubeInstance getInstance() { - return this.instance; - } - - public void setInstance(PeertubeInstance instance) { - this.instance = instance; + return getInstance().getUrl(); } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index c0e8721250..eb14a67257 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -9,10 +9,13 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.localization.Localization; +import org.schabi.newpipe.extractor.services.youtube.invidious.InvidiousInstance; import org.schabi.newpipe.extractor.stream.Description; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Utils; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; @@ -22,15 +25,7 @@ import java.time.OffsetDateTime; import java.time.ZoneOffset; import java.time.format.DateTimeParseException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; - -import javax.annotation.Nonnull; -import javax.annotation.Nullable; +import java.util.*; import static org.schabi.newpipe.extractor.NewPipe.getDownloader; import static org.schabi.newpipe.extractor.utils.Utils.*; @@ -84,41 +79,43 @@ private static boolean isGoogleURL(String url) { public static boolean isYoutubeURL(final URL url) { final String host = url.getHost(); - return host.equalsIgnoreCase("youtube.com") || host.equalsIgnoreCase("www.youtube.com") - || host.equalsIgnoreCase("m.youtube.com") || host.equalsIgnoreCase("music.youtube.com"); + return host.equalsIgnoreCase("youtube.com") + || host.equalsIgnoreCase("www.youtube.com") + || host.equalsIgnoreCase("m.youtube.com") + || host.equalsIgnoreCase("music.youtube.com"); } - public static boolean isYoutubeServiceURL(final URL url) { + public static boolean isYoutubeServiceUrl(final URL url) { final String host = url.getHost(); return host.equalsIgnoreCase("www.youtube-nocookie.com") || host.equalsIgnoreCase("youtu.be"); } public static boolean isHooktubeURL(final URL url) { - final String host = url.getHost(); - return host.equalsIgnoreCase("hooktube.com"); + return url.getHost().equalsIgnoreCase("hooktube.com"); } - public static boolean isInvidioURL(final URL url) { + public static boolean isInvidiousURL(final URL url) { + if (isBlank(url.getAuthority())) { + return false; + } + + final InvidiousInstance instance = new InvidiousInstance(Utils.getBaseUrl(url)); + return instance.isValid(); + } + + /** + * Test if URL is an Invidious-redirect URL + * + * @param url the URL to test + * @return if the URL is an Invidious-redirect URL + * @see Invidious-redirect + */ + public static boolean isInvidiousRedirectUrl(final URL url) { final String host = url.getHost(); - return host.equalsIgnoreCase("invidio.us") - || host.equalsIgnoreCase("dev.invidio.us") + return host.equalsIgnoreCase("redirect.invidious.io") + || host.equalsIgnoreCase("invidio.us") || host.equalsIgnoreCase("www.invidio.us") - || host.equalsIgnoreCase("redirect.invidious.io") - || host.equalsIgnoreCase("invidious.snopyta.org") - || host.equalsIgnoreCase("yewtu.be") - || host.equalsIgnoreCase("tube.connect.cafe") - || host.equalsIgnoreCase("invidious.zapashcanon.fr") - || host.equalsIgnoreCase("invidious.kavin.rocks") - || host.equalsIgnoreCase("invidious.tube") - || host.equalsIgnoreCase("invidious.site") - || host.equalsIgnoreCase("invidious.xyz") - || host.equalsIgnoreCase("vid.mint.lgbt") - || host.equalsIgnoreCase("invidiou.site") - || host.equalsIgnoreCase("invidious.fdn.fr") - || host.equalsIgnoreCase("invidious.048596.xyz") - || host.equalsIgnoreCase("invidious.zee.li") - || host.equalsIgnoreCase("vid.puffyan.us") - || host.equalsIgnoreCase("ytprivate.com"); + || host.equalsIgnoreCase("dev.invidio.us"); } /** @@ -453,7 +450,6 @@ public static String[] getYoutubeMusicKeys() throws IOException, ReCaptchaExcept } - @Nullable public static String getUrlFromNavigationEndpoint(JsonObject navigationEndpoint) throws ParsingException { if (navigationEndpoint.has("urlEndpoint")) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java index 6e7ab74070..7152904f82 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java @@ -76,7 +76,7 @@ public YoutubeService(int id) { @Override public String getBaseUrl() { - return "https://youtube.com"; + return "https://www.youtube.com"; } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java index 5f876e719b..f5281ae292 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java @@ -187,7 +187,7 @@ private void collectStreamsFrom( } } - private String getThumbnailUrlFromPlaylistId(final String playlistId) throws ParsingException { + public static String getThumbnailUrlFromPlaylistId(final String playlistId) throws ParsingException { final String videoId; if (playlistId.startsWith("RDMM")) { videoId = playlistId.substring(4); @@ -202,7 +202,7 @@ private String getThumbnailUrlFromPlaylistId(final String playlistId) throws Par return getThumbnailUrlFromVideoId(videoId); } - private String getThumbnailUrlFromVideoId(final String videoId) { + private static String getThumbnailUrlFromVideoId(final String videoId) { return "https://i.ytimg.com/vi/" + videoId + "/hqdefault.jpg"; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 39418f6122..3f1ba14ac3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -929,8 +929,8 @@ private Map getItags(final String streamingDataKey, } final JsonArray formats = streamingData.getArray(streamingDataKey); - for (int i = 0; i != formats.size(); ++i) { - JsonObject formatData = formats.getObject(i); + for (Object o : formats) { + JsonObject formatData = (JsonObject) o; int itag = formatData.getInt("itag"); if (ItagItem.isSupported(itag)) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousInstance.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousInstance.java new file mode 100644 index 0000000000..bb1da125f6 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousInstance.java @@ -0,0 +1,113 @@ +package org.schabi.newpipe.extractor.services.youtube.invidious; + +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; +import org.schabi.newpipe.extractor.Instance; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.downloader.Response; +import org.schabi.newpipe.extractor.exceptions.InvalidInstanceException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; +import org.schabi.newpipe.extractor.utils.JsonUtils; +import org.schabi.newpipe.extractor.utils.Utils; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; + +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*; + +public class InvidiousInstance implements Instance { + + private final String url; + private String name; + private Boolean isValid = null; + + private static final String STATISTICS_DISABLED = "Statistics are not enabled."; + + private static InvidiousInstance defaultInstance = new InvidiousInstance("https://invidious.fdn.fr"); + + public InvidiousInstance(String url, String name) { + this.url = url; + this.name = name; + } + + public InvidiousInstance(String url) { + this(url, "invidious"); + } + + @Nullable + @Override + public String getName() { + return name; + } + + @Nonnull + @Override + public String getUrl() { + return url; + } + + @Override + public boolean isValid() { + if (isValid != null) { + return isValid; + } + + URL url; + try { + url = Utils.stringToURL(this.url); + } catch (MalformedURLException e) { + return false; + } + + if (isYoutubeURL(url) || isHooktubeURL(url) || isInvidiousRedirectUrl(url)) { + return isValid = false; + } + + try { + fetchInstanceMetaData(); + return isValid = true; + } catch (InvalidInstanceException e) { + return isValid = false; + } + } + + @Override + public void fetchInstanceMetaData() throws InvalidInstanceException { + final Downloader downloader = NewPipe.getDownloader(); + Response response; + + try { + response = downloader.get(url + "/api/v1/stats?fields=software,error"); + } catch (ReCaptchaException | IOException | IllegalArgumentException e) { + throw new InvalidInstanceException("unable to configure instance " + url, e); + } + + if (response == null || Utils.isBlank(response.responseBody())) { + throw new InvalidInstanceException("unable to configure instance " + url); + } + + try { + JsonObject json = JsonParser.object().from(response.responseBody()); + if (json.has("software")) { + this.name = JsonUtils.getString(json, "software.name"); + } else if (json.has("error")) { + if (!STATISTICS_DISABLED.equals(json.getString("error"))) { + throw new ParsingException("Could not get stats from instance " + url); + } + } + } catch (JsonParserException | ParsingException e) { + throw new InvalidInstanceException("unable to parse instance config", e); + } + } + + public static InvidiousInstance getDefaultInstance() { + return defaultInstance; + } + +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousParsingHelper.java new file mode 100644 index 0000000000..dba6ecb100 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousParsingHelper.java @@ -0,0 +1,141 @@ +package org.schabi.newpipe.extractor.services.youtube.invidious; + +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; +import org.schabi.newpipe.extractor.Page; +import org.schabi.newpipe.extractor.downloader.Response; +import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.localization.DateWrapper; + +import javax.annotation.Nonnull; +import java.util.Calendar; +import java.util.Date; + +import static org.schabi.newpipe.extractor.utils.Utils.HTTP; +import static org.schabi.newpipe.extractor.utils.Utils.HTTPS; + +public class InvidiousParsingHelper { + + + /** + * Get valid JsonObject + *

+ * Checks status code and handle JSON parsing + * + * @param response the response got from the service + * @param apiUrl the url used to call the service + * @return the JsonObject + * @throws ExtractionException if the HTTP code indicate an error or the json parsing went wrong. + */ + public static JsonObject getValidJsonObjectFromResponse(final Response response, final String apiUrl) throws ExtractionException { + final String responseBody = getValidResponseBody(response, apiUrl); + + try { + return JsonParser.object().from(responseBody); + } catch (JsonParserException e) { + throw new ExtractionException("Could not parse json from page \"" + apiUrl + "\"", e); + } + } + + /** + * Get valid Response body + *

+ * Checks status code and handle JSON parsing + * + * @param response the response got from the service + * @param apiUrl the url used to call the service + * @return the response body + * @throws ExtractionException if the HTTP code indicate an error or the json parsing went wrong. + */ + public static String getValidResponseBody(final Response response, final String apiUrl) throws ExtractionException { + if (response.responseCode() == 404) { + throw new ContentNotAvailableException("Could not get page " + apiUrl + " (HTTP " + response.responseCode() + " : " + response.responseMessage()); + } else if (response.responseCode() >= 400) { + throw new ExtractionException("Could not get page " + apiUrl + " (HTTP " + response.responseCode() + " : " + response.responseMessage()); + } + + return response.responseBody(); + } + + /** + * Get valid Response body + *

+ * Checks status code and handle JSON parsing + * + * @param response the response got from the service + * @param apiUrl the url used to call the service + * @return the JsonArray + * @throws ExtractionException if the HTTP code indicate an error or the json parsing went wrong. + */ + public static JsonArray getValidJsonArrayFromResponse(final Response response, final String apiUrl) throws ExtractionException { + final String responseBody = getValidResponseBody(response, apiUrl); + + try { + return JsonParser.array().from(responseBody); + } catch (JsonParserException e) { + throw new ExtractionException("Could not parse json from page \"" + apiUrl + "\"", e); + } + } + + public static DateWrapper getUploadDateFromEpochTime(final long epochTime) { + Calendar calendar = Calendar.getInstance(); + calendar.setTime(new Date(epochTime * 1000)); // * 1000 because it's second-based, not millisecond based + return new DateWrapper(calendar); + } + + public static String getUid(@Nonnull String id) { + if (id.startsWith("user/")) { + id = id.substring(5); + } else if (id.startsWith("channel/")) { + id = id.substring(8); + } + + return id; + } + + public static Page getPage(final String url, final int page) { + return new Page(url + "?page=" + page, String.valueOf(page)); + } + + /** + * Get thumbnail URL at a reasonable quality + * + * @param thumbnails an array of thumbnails + * @return a thumbnail URL at a reasonable quality + */ + public static String getThumbnailUrl(final JsonArray thumbnails) { + String url = ""; + + if (thumbnails.size() <= 0) { + return url; + } else if (thumbnails.size() == 1) { + url = thumbnails.getObject(0).getString("url"); + } else { + url = thumbnails.getObject(thumbnails.size() - 1).getString("url"); + for (int i = 1; i < thumbnails.size(); i++) { + JsonObject thumbnail = thumbnails.getObject(i); + String quality = thumbnail.getString("quality"); + if ("high".equals(quality)) { + url = thumbnail.getString("url"); + break; + } + } + } + return fixThumbnailUrl(url); + } + + public static String fixThumbnailUrl(String thumbnailUrl) { + if (thumbnailUrl.startsWith(HTTP) || thumbnailUrl.startsWith(HTTPS)) { + return thumbnailUrl; + } + + if (thumbnailUrl.startsWith("//")) { + thumbnailUrl = thumbnailUrl.substring(2); + } + + return HTTPS + thumbnailUrl; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousService.java new file mode 100644 index 0000000000..af46394a81 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousService.java @@ -0,0 +1,180 @@ +package org.schabi.newpipe.extractor.services.youtube.invidious; + +import org.schabi.newpipe.extractor.Instance; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.channel.ChannelExtractor; +import org.schabi.newpipe.extractor.comments.CommentsExtractor; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.feed.FeedExtractor; +import org.schabi.newpipe.extractor.kiosk.KioskList; +import org.schabi.newpipe.extractor.linkhandler.*; +import org.schabi.newpipe.extractor.localization.ContentCountry; +import org.schabi.newpipe.extractor.localization.Localization; +import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; +import org.schabi.newpipe.extractor.search.SearchExtractor; +import org.schabi.newpipe.extractor.services.youtube.YoutubeService; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSubscriptionExtractor; +import org.schabi.newpipe.extractor.services.youtube.invidious.extractors.*; +import org.schabi.newpipe.extractor.services.youtube.invidious.linkhandler.InvidiousSearchQueryHandlerFactory; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.*; +import org.schabi.newpipe.extractor.stream.StreamExtractor; +import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; +import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor; + +import javax.annotation.Nonnull; +import java.util.List; + +import static java.util.Arrays.asList; +import static org.schabi.newpipe.extractor.ServiceList.YouTube; +import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.*; + +/** + * InvidiousService, uses documented API: https://github.com/iv-org/documentation/blob/master/API.md + */ +public class InvidiousService extends StreamingService { + + public InvidiousService(int id) { + this(id, InvidiousInstance.getDefaultInstance()); + } + + public InvidiousService(final int id, final Instance instance) { + super(id, "Invidious", asList(AUDIO, VIDEO, LIVE, COMMENTS, INSTANCES)); + setInstance(instance); + } + + @Override + public String getBaseUrl() { + return getInstance().getUrl(); + } + + @Override + public LinkHandlerFactory getStreamLHFactory() { + return YoutubeStreamLinkHandlerFactory.getInstance(); + } + + @Override + public ListLinkHandlerFactory getChannelLHFactory() { + return YoutubeChannelLinkHandlerFactory.getInstance(); + } + + @Override + public ListLinkHandlerFactory getPlaylistLHFactory() { + return YoutubePlaylistLinkHandlerFactory.getInstance(); + } + + @Override + public SearchQueryHandlerFactory getSearchQHFactory() { + return InvidiousSearchQueryHandlerFactory.getInstance(getBaseUrl()); + } + + @Override + public StreamExtractor getStreamExtractor(LinkHandler linkHandler) { + return new InvidiousStreamExtractor(this, linkHandler); + } + + @Override + public ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler) { + return new InvidiousChannelExtractor(this, linkHandler); + } + + @Override + public PlaylistExtractor getPlaylistExtractor(ListLinkHandler linkHandler) { + return new InvidiousPlaylistExtractor(this, linkHandler); + } + + @Override + public SearchExtractor getSearchExtractor(SearchQueryHandler query) { + final List contentFilters = query.getContentFilters(); + + if (contentFilters.size() > 0 && contentFilters.get(0).startsWith("music_")) { + return null; // use YoutubeMusicSearchExtractor? + } else { + return new InvidiousSearchExtractor(this, query); + } + } + + @Override + public SuggestionExtractor getSuggestionExtractor() { + return new InvidiousSuggestionExtractor(this); + } + + @Override + public KioskList getKioskList() throws ExtractionException { + KioskList list = new KioskList(this); + + try { + list.addKioskEntry((streamingService, url, id) -> new InvidiousTrendingExtractor(InvidiousService.this, + new YoutubeTrendingLinkHandlerFactory().fromUrl(url), id), new YoutubeTrendingLinkHandlerFactory(), "Trending"); + list.setDefaultKiosk("Trending"); + } catch (Exception e) { + throw new ExtractionException(e); + } + + return list; + } + + @Override + public SubscriptionExtractor getSubscriptionExtractor() { + return new YoutubeSubscriptionExtractor(YouTube); + } + + @Nonnull + @Override + public FeedExtractor getFeedExtractor(final String channelUrl) throws ExtractionException { + return new InvidiousFeedExtractor(this, getChannelLHFactory().fromUrl(channelUrl)); + } + + @Override + public ListLinkHandlerFactory getCommentsLHFactory() { + return YoutubeCommentsLinkHandlerFactory.getInstance(); + } + + @Override + public CommentsExtractor getCommentsExtractor(ListLinkHandler urlIdHandler) throws ExtractionException { + return new InvidiousCommentsExtractor(this, urlIdHandler); + } + + /*////////////////////////////////////////////////////////////////////////// + // Localization + //////////////////////////////////////////////////////////////////////////*/ + + // https://www.youtube.com/picker_ajax?action_language_json=1 + private static final List SUPPORTED_LANGUAGES = Localization.listFrom( + "en-GB" + /*"af", "am", "ar", "az", "be", "bg", "bn", "bs", "ca", "cs", "da", "de", + "el", "en", "en-GB", "es", "es-419", "es-US", "et", "eu", "fa", "fi", "fil", "fr", + "fr-CA", "gl", "gu", "hi", "hr", "hu", "hy", "id", "is", "it", "iw", "ja", + "ka", "kk", "km", "kn", "ko", "ky", "lo", "lt", "lv", "mk", "ml", "mn", + "mr", "ms", "my", "ne", "nl", "no", "pa", "pl", "pt", "pt-PT", "ro", "ru", + "si", "sk", "sl", "sq", "sr", "sr-Latn", "sv", "sw", "ta", "te", "th", "tr", + "uk", "ur", "uz", "vi", "zh-CN", "zh-HK", "zh-TW", "zu"*/ + ); + + // https://www.youtube.com/picker_ajax?action_country_json=1 + private static final List SUPPORTED_COUNTRIES = ContentCountry.listFrom( + "AD", "AE", "AF", "AG", "AI", "AL", "AM", "AO", "AQ", "AR", "AS", "AT", "AU", "AW", "AX", "AZ", "BA", + "BB", "BD", "BE", "BF", "BG", "BH", "BI", "BJ", "BL", "BM", "BN", "BO", "BQ", "BR", "BS", "BT", "BV", + "BW", "BY", "BZ", "CA", "CC", "CD", "CF", "CG", "CH", "CI", "CK", "CL", "CM", "CN", "CO", "CR", "CU", + "CV", "CW", "CX", "CY", "CZ", "DE", "DJ", "DK", "DM", "DO", "DZ", "EC", "EE", "EG", "EH", "ER", "ES", + "ET", "FI", "FJ", "FK", "FM", "FO", "FR", "GA", "GB", "GD", "GE", "GF", "GG", "GH", "GI", "GL", "GM", + "GN", "GP", "GQ", "GR", "GS", "GT", "GU", "GW", "GY", "HK", "HM", "HN", "HR", "HT", "HU", "ID", "IE", + "IL", "IM", "IN", "IO", "IQ", "IR", "IS", "IT", "JE", "JM", "JO", "JP", "KE", "KG", "KH", "KI", "KM", + "KN", "KP", "KR", "KW", "KY", "KZ", "LA", "LB", "LC", "LI", "LK", "LR", "LS", "LT", "LU", "LV", "LY", + "MA", "MC", "MD", "ME", "MF", "MG", "MH", "MK", "ML", "MM", "MN", "MO", "MP", "MQ", "MR", "MS", "MT", + "MU", "MV", "MW", "MX", "MY", "MZ", "NA", "NC", "NE", "NF", "NG", "NI", "NL", "NO", "NP", "NR", "NU", + "NZ", "OM", "PA", "PE", "PF", "PG", "PH", "PK", "PL", "PM", "PN", "PR", "PS", "PT", "PW", "PY", "QA", + "RE", "RO", "RS", "RU", "RW", "SA", "SB", "SC", "SD", "SE", "SG", "SH", "SI", "SJ", "SK", "SL", "SM", + "SN", "SO", "SR", "SS", "ST", "SV", "SX", "SY", "SZ", "TC", "TD", "TF", "TG", "TH", "TJ", "TK", "TL", + "TM", "TN", "TO", "TR", "TT", "TV", "TW", "TZ", "UA", "UG", "UM", "US", "UY", "UZ", "VA", "VC", "VE", + "VG", "VI", "VN", "VU", "WF", "WS", "YE", "YT", "ZA", "ZM", "ZW" + ); + + @Override + public List getSupportedLocalizations() { + return SUPPORTED_LANGUAGES; + } + + public List getSupportedCountries() { + return SUPPORTED_COUNTRIES; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousChannelExtractor.java new file mode 100644 index 0000000000..7c53dfd79e --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousChannelExtractor.java @@ -0,0 +1,152 @@ +package org.schabi.newpipe.extractor.services.youtube.invidious.extractors; + +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.Page; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.channel.ChannelExtractor; +import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.downloader.Response; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.services.youtube.invidious.InvidiousParsingHelper; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; + +import javax.annotation.Nonnull; +import java.io.IOException; + +import static org.schabi.newpipe.extractor.services.youtube.invidious.InvidiousParsingHelper.getUid; + +/* + * Copyright (C) 2020 Team NewPipe + * InvidiousChannelExtractor.java is part of NewPipe Extractor. + * + * NewPipe Extractor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe Extractor. If not, see . + */ + +public class InvidiousChannelExtractor extends ChannelExtractor { + + private final String baseUrl; + private JsonObject json; + + public InvidiousChannelExtractor(StreamingService service, ListLinkHandler linkHandler) { + super(service, linkHandler); + this.baseUrl = service.getInstance().getUrl(); + } + + @Override + public String getAvatarUrl() { + return InvidiousParsingHelper.getThumbnailUrl(json.getArray("authorThumbnails")); + } + + @Override + public String getBannerUrl() { + return InvidiousParsingHelper.getThumbnailUrl(json.getArray("authorBanners")); + } + + @Override + public String getFeedUrl() { + return baseUrl + "/feed/channel/" + json.getString("authorId"); + } + + @Override + public long getSubscriberCount() { + return json.getNumber("subCount").longValue(); + } + + @Override + public String getDescription() { + return json.getString("description"); + } + + @Override + public String getParentChannelName() { + return null; + } + + @Override + public String getParentChannelUrl() { + return null; + } + + @Override + public String getParentChannelAvatarUrl() { + return null; + } + + @Override + public boolean isVerified() throws ParsingException { + return false; + } + + @Nonnull + @Override + public InfoItemsPage getInitialPage() throws IOException, ExtractionException { + return getPage(getPage(1)); + } + + public Page getPage(int page) { + return InvidiousParsingHelper.getPage(baseUrl + "/api/v1/channels/videos/" + + json.getString("authorId"), page); + } + + @Override + public InfoItemsPage getPage(Page page) throws IOException, ExtractionException { + final Downloader dl = NewPipe.getDownloader(); + final String apiUrl = page.getUrl(); + final Response rp = dl.get(apiUrl); + final JsonArray array = InvidiousParsingHelper.getValidJsonArrayFromResponse(rp, apiUrl); + + final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + collectStreamsFrom(collector, array); + + Page nextPage; + if (array.size() < 59) { + // max number of items per page + // with Second it is 29 but next Page logic is not implemented + + nextPage = null; + } else { + nextPage = getPage(Integer.parseInt(page.getId()) + 1); + } + + return new InfoItemsPage<>(collector, nextPage); + } + + @Override + public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { + final String apiUrl = baseUrl + "/api/v1/channels/" + getUid(getId()) + + "?fields=author,description,subCount,authorThumbnails,authorBanners,authorId" + + "®ion=" + getExtractorContentCountry().getCountryCode(); + + final Response response = downloader.get(apiUrl); + + json = InvidiousParsingHelper.getValidJsonObjectFromResponse(response, apiUrl); + } + + @Nonnull + @Override + public String getName() { + return json.getString("author"); + } + + private void collectStreamsFrom(final StreamInfoItemsCollector collector, final JsonArray videos) { + for (Object o : videos) { + collector.commit(new InvidiousStreamInfoItemExtractor((JsonObject) o, baseUrl)); + } + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousChannelInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousChannelInfoItemExtractor.java new file mode 100644 index 0000000000..ead263a208 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousChannelInfoItemExtractor.java @@ -0,0 +1,52 @@ +package org.schabi.newpipe.extractor.services.youtube.invidious.extractors; + +import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.services.youtube.invidious.InvidiousParsingHelper; + +public class InvidiousChannelInfoItemExtractor implements ChannelInfoItemExtractor { + + private final JsonObject json; + private final String baseUrl; + + public InvidiousChannelInfoItemExtractor(final JsonObject json, final String baseUrl) { + this.json = json; + this.baseUrl = baseUrl; + } + + @Override + public String getName() { + return json.getString("author"); + } + + @Override + public String getUrl() { + return baseUrl + json.getString("authorUrl"); + } + + @Override + public String getThumbnailUrl() { + return InvidiousParsingHelper.getThumbnailUrl(json.getArray("authorThumbnails")); + } + + @Override + public String getDescription() throws ParsingException { + return json.getString("description"); // descriptionHtml is also available + } + + @Override + public long getSubscriberCount() throws ParsingException { + return json.getLong("subCount"); + } + + @Override + public long getStreamCount() throws ParsingException { + return json.getLong("videoCount"); + } + + @Override + public boolean isVerified() throws ParsingException { + return false; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousCommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousCommentsExtractor.java new file mode 100644 index 0000000000..06219c3c3d --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousCommentsExtractor.java @@ -0,0 +1,92 @@ +package org.schabi.newpipe.extractor.services.youtube.invidious.extractors; + +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.Page; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.comments.CommentsExtractor; +import org.schabi.newpipe.extractor.comments.CommentsInfoItem; +import org.schabi.newpipe.extractor.comments.CommentsInfoItemsCollector; +import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.downloader.Response; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.services.youtube.invidious.InvidiousParsingHelper; + +import javax.annotation.Nonnull; +import java.io.IOException; + +/* + * Copyright (C) 2020 Team NewPipe + * InvidiousCommentsExtractor.java is part of NewPipe Extractor. + * + * NewPipe Extractor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe Extractor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe Extractor. If not, see . + */ + +public class InvidiousCommentsExtractor extends CommentsExtractor { + + private final String baseUrl; + private JsonObject json; + + public InvidiousCommentsExtractor(StreamingService service, ListLinkHandler uiHandler) { + super(service, uiHandler); + baseUrl = service.getInstance().getUrl(); + } + + @Nonnull + @Override + public InfoItemsPage getInitialPage() throws ExtractionException { + final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId()); + + collectStreamsFrom(collector, json.getArray("comments"), getUrl()); + + return new InfoItemsPage<>(collector, getNextPage()); + } + + @Override + public InfoItemsPage getPage(Page page) throws IOException, ExtractionException { + final Downloader dl = NewPipe.getDownloader(); + final Response response = dl.get(page.getUrl()); + + json = InvidiousParsingHelper.getValidJsonObjectFromResponse(response, page.getUrl()); + + final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId()); + collectStreamsFrom(collector, json.getArray("comments"), page.getUrl()); + return new InfoItemsPage<>(collector, getNextPage()); + } + + + public Page getNextPage() throws ParsingException { + return new Page(baseUrl + "/api/v1/comments/" + getId() + + "?continuation=" + json.getString("continuation")); + } + + @Override + public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { + final String apiUrl = baseUrl + "/api/v1/comments/" + getId(); + final Response response = downloader.get(apiUrl); + + json = InvidiousParsingHelper.getValidJsonObjectFromResponse(response, apiUrl); + + } + + private void collectStreamsFrom(final CommentsInfoItemsCollector collector, final JsonArray entries, final String url) { + for (Object comment : entries) { + collector.commit(new InvidiousCommentsInfoItemExtractor((JsonObject) comment, url)); + } + } + +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousCommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousCommentsInfoItemExtractor.java new file mode 100644 index 0000000000..14deb89cbc --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousCommentsInfoItemExtractor.java @@ -0,0 +1,93 @@ +package org.schabi.newpipe.extractor.services.youtube.invidious.extractors; + +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.comments.CommentsInfoItemExtractor; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.localization.DateWrapper; +import org.schabi.newpipe.extractor.services.youtube.invidious.InvidiousParsingHelper; + +import javax.annotation.Nullable; + +public class InvidiousCommentsInfoItemExtractor implements CommentsInfoItemExtractor { + + private final JsonObject json; + private final String url; + + public InvidiousCommentsInfoItemExtractor(final JsonObject json, final String url) { + this.json = json; + this.url = url; + } + + @Override + public int getLikeCount() { + return json.getNumber("likeCount").intValue(); + } + + @Override + public String getCommentText() { + return json.getString("content"); + } + + @Override + public String getTextualUploadDate() { + return json.getString("publishedText"); + } + + @Nullable + @Override + public DateWrapper getUploadDate() { + return InvidiousParsingHelper.getUploadDateFromEpochTime(json.getNumber("published").longValue()); + } + + @Override + public String getCommentId() { + return json.getString("commentId"); + } + + @Override + public String getUploaderUrl() { + return json.getString("authorUrl"); + } + + @Override + public String getUploaderName() { + return json.getString("author"); + } + + @Override + public String getUploaderAvatarUrl() { + return InvidiousParsingHelper.getThumbnailUrl(json.getArray("authorThumbnails")); + } + + @Override + public boolean isHeartedByUploader() throws ParsingException { + return json.has("creatorHeart"); + } + + @Override + public boolean isPinned() throws ParsingException { + return false; + } + + @Override + public boolean isUploaderVerified() throws ParsingException { + return false; + } + + @Override + public String getName() throws ParsingException { + return json.getString("author"); + } + + @Override + public String getUrl() throws ParsingException { + return url; + } + + @Override + public String getThumbnailUrl() { + return InvidiousParsingHelper.getThumbnailUrl(json.getArray("authorThumbnails")); + } + +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousFeedExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousFeedExtractor.java new file mode 100644 index 0000000000..e34dccc65e --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousFeedExtractor.java @@ -0,0 +1,78 @@ +package org.schabi.newpipe.extractor.services.youtube.invidious.extractors; + +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; +import org.schabi.newpipe.extractor.Page; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.downloader.Response; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.feed.FeedExtractor; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeFeedInfoItemExtractor; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; + +import javax.annotation.Nonnull; +import java.io.IOException; + +import static org.schabi.newpipe.extractor.services.youtube.invidious.InvidiousParsingHelper.getValidResponseBody; + +public class InvidiousFeedExtractor extends FeedExtractor { + + private final String baseUrl; + private Document document; + + public InvidiousFeedExtractor(StreamingService service, ListLinkHandler listLinkHandler) { + super(service, listLinkHandler); + this.baseUrl = service.getInstance().getUrl(); + } + + @Nonnull + @Override + public InfoItemsPage getInitialPage() { + final Elements entries = document.select("feed > entry"); + final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + + for (Element entryElement : entries) { + collector.commit(new YoutubeFeedInfoItemExtractor(entryElement)); + // no need for InvidiousFeedInfoItemExtractor, it's exactly the same structure + } + + return new InfoItemsPage<>(collector, null); + + } + + @Override + public InfoItemsPage getPage(Page page) throws IOException, ExtractionException { + return null; + } + + @Override + public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { + final String feedUrl = baseUrl + "/feed/channel/" + getLinkHandler().getId(); + final Response response = downloader.get(feedUrl); + document = Jsoup.parse(getValidResponseBody(response, feedUrl)); + } + + @Nonnull + @Override + public String getName() throws ParsingException { + return document.select("feed > author > name").first().text(); + } + + @Nonnull + @Override + public String getUrl() throws ParsingException { + return document.select("feed > author > uri").first().text(); + } + + @Nonnull + @Override + public String getId() throws ParsingException { + return document.getElementsByTag("yt:channelId").first().text(); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousMixPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousMixPlaylistExtractor.java new file mode 100644 index 0000000000..c46c334b25 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousMixPlaylistExtractor.java @@ -0,0 +1,115 @@ +package org.schabi.newpipe.extractor.services.youtube.invidious.extractors; + +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.ListExtractor; +import org.schabi.newpipe.extractor.Page; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.downloader.Response; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMixPlaylistExtractor; +import org.schabi.newpipe.extractor.services.youtube.invidious.InvidiousParsingHelper; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; + +import javax.annotation.Nonnull; +import java.io.IOException; + +public class InvidiousMixPlaylistExtractor extends PlaylistExtractor { + + private JsonObject json; + private final String baseUrl; + + public InvidiousMixPlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) { + super(service, linkHandler); + this.baseUrl = service.getInstance().getUrl(); + } + + @Override + public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { + final String apiUrl = baseUrl + "/api/v1/mixes/" + getId(); + final Response response = downloader.get(apiUrl); + + json = InvidiousParsingHelper.getValidJsonObjectFromResponse(response, apiUrl); + } + + @Nonnull + @Override + public String getName() throws ParsingException { + return json.getString("title"); + } + + @Nonnull + @Override + public InfoItemsPage getInitialPage() throws IOException, ExtractionException { + final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + final JsonArray videos = json.getArray("videos"); + for (Object o : videos) { + final JsonObject video = (JsonObject) o; + collector.commit(new InvidiousStreamInfoItemExtractor(video, baseUrl)); + } + return new InfoItemsPage<>(collector, null); + } + + @Override + public InfoItemsPage getPage(Page page) throws IOException, ExtractionException { + return InfoItemsPage.emptyPage(); + } + + @Override + public String getThumbnailUrl() throws ParsingException { + return YoutubeMixPlaylistExtractor.getThumbnailUrlFromPlaylistId(json.getString("mixId")); + } + + @Override + public String getBannerUrl() throws ParsingException { + return ""; + } + + @Override + public String getUploaderUrl() throws ParsingException { + return ""; + } + + @Override + public String getUploaderName() throws ParsingException { + return "YouTube"; + } + + @Override + public String getUploaderAvatarUrl() throws ParsingException { + return ""; + } + + @Override + public boolean isUploaderVerified() throws ParsingException { + return false; + } + + @Override + public long getStreamCount() throws ParsingException { + return ListExtractor.ITEM_COUNT_INFINITE; + } + + @Nonnull + @Override + public String getSubChannelName() throws ParsingException { + return ""; + } + + @Nonnull + @Override + public String getSubChannelUrl() throws ParsingException { + return ""; + } + + @Nonnull + @Override + public String getSubChannelAvatarUrl() throws ParsingException { + return ""; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousPlaylistExtractor.java new file mode 100644 index 0000000000..5c1440587d --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousPlaylistExtractor.java @@ -0,0 +1,161 @@ +package org.schabi.newpipe.extractor.services.youtube.invidious.extractors; + +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.Page; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.downloader.Response; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; +import org.schabi.newpipe.extractor.services.youtube.invidious.InvidiousParsingHelper; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; + +import javax.annotation.Nonnull; +import java.io.IOException; + +/* + * Copyright (C) 2020 Team NewPipe + * InvidiousPlaylistExtractor.java is part of NewPipe Extractor. + * + * NewPipe Extractor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe Extractor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe Extractor. If not, see . + */ + +public class InvidiousPlaylistExtractor extends PlaylistExtractor { + + private JsonObject json; + private final String baseUrl; + + public InvidiousPlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) { + super(service, linkHandler); + this.baseUrl = service.getInstance().getUrl(); + } + + @Override + public String getThumbnailUrl() { + final JsonArray thumbnails = json.getArray("authorThumbnails"); + return InvidiousParsingHelper.getThumbnailUrl(thumbnails); + } + + @Override + public String getBannerUrl() { + return null; // should it be ""? + } + + @Override + public String getUploaderUrl() { + return baseUrl + "/channel/" + json.getString("authorId"); + } + + @Override + public String getUploaderName() { + return json.getString("author"); + } + + @Override + public String getUploaderAvatarUrl() { + return InvidiousParsingHelper.getThumbnailUrl(json.getArray("authorThumbnails")); + } + + @Override + public boolean isUploaderVerified() throws ParsingException { + return false; + } + + @Override + public long getStreamCount() { + final Number number = json.getNumber("videoCount"); + return number == null ? -1 : number.longValue(); + } + + @Nonnull + @Override + public String getSubChannelName() { + return ""; + } + + @Nonnull + @Override + public String getSubChannelUrl() { + return ""; + } + + @Nonnull + @Override + public String getSubChannelAvatarUrl() { + return ""; + } + + @Nonnull + @Override + public InfoItemsPage getInitialPage() throws IOException, ExtractionException { + return getPage(getPage(1)); + } + + @Override + public InfoItemsPage getPage(Page page) throws IOException, ExtractionException { + if (Integer.parseInt(page.getId()) != 1) { + final Downloader dl = NewPipe.getDownloader(); + final String apiUrl = page.getUrl(); + final Response rp = dl.get(apiUrl); + json = InvidiousParsingHelper.getValidJsonObjectFromResponse(rp, apiUrl); + } + + final JsonArray videos = json.getArray("videos"); + + final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + collectStreamsFrom(collector, videos); + + Page nextPage; + if (videos.size() < 99) { + // max number of items per page + nextPage = null; + } else { + nextPage = getPage(Integer.parseInt(page.getId()) + 1); + } + + return new InfoItemsPage<>(collector, nextPage); + } + + public Page getPage(int page) throws ParsingException { + return InvidiousParsingHelper.getPage(baseUrl + "/api/v1/playlists/" + + getId(), page); + } + + @Override + public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { + final String apiUrl = baseUrl + "/api/v1/playlists/" + getId(); + + final Response response = downloader.get(apiUrl); + + json = InvidiousParsingHelper.getValidJsonObjectFromResponse(response, apiUrl); + } + + @Nonnull + @Override + public String getName() throws ParsingException { + return json.getString("title"); + } + + private void collectStreamsFrom(StreamInfoItemsCollector collector, JsonArray videos) { + for (Object o : videos) { + collector.commit(new InvidiousStreamInfoItemExtractor((JsonObject) o, baseUrl)); + } + } + +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousPlaylistInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousPlaylistInfoItemExtractor.java new file mode 100644 index 0000000000..990fbb3f97 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousPlaylistInfoItemExtractor.java @@ -0,0 +1,42 @@ +package org.schabi.newpipe.extractor.services.youtube.invidious.extractors; + +import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor; +import org.schabi.newpipe.extractor.services.youtube.invidious.InvidiousParsingHelper; + +public class InvidiousPlaylistInfoItemExtractor implements PlaylistInfoItemExtractor { + + private final JsonObject json; + private final String baseUrl; + + public InvidiousPlaylistInfoItemExtractor(final JsonObject json, final String baseUrl) { + this.json = json; + this.baseUrl = baseUrl; + } + + @Override + public String getName() { + return json.getString("title"); + } + + @Override + public String getUrl() { + return baseUrl + "/playlist?list=" + json.getString("playlistId"); + } + + @Override + public String getThumbnailUrl() { + return InvidiousParsingHelper.getThumbnailUrl(json.getArray("videos").getObject(0).getArray("videoThumbnails")); + } + + @Override + public String getUploaderName() throws ParsingException { + return json.getString("author"); + } + + @Override + public long getStreamCount() throws ParsingException { + return json.getLong("videoCount"); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousSearchExtractor.java new file mode 100644 index 0000000000..e56e1f054d --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousSearchExtractor.java @@ -0,0 +1,90 @@ +package org.schabi.newpipe.extractor.services.youtube.invidious.extractors; + +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.InfoItem; +import org.schabi.newpipe.extractor.MetaInfo; +import org.schabi.newpipe.extractor.Page; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.downloader.Response; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; +import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector; +import org.schabi.newpipe.extractor.search.SearchExtractor; +import org.schabi.newpipe.extractor.services.youtube.invidious.InvidiousParsingHelper; + +import javax.annotation.Nonnull; +import java.io.IOException; +import java.util.Collections; +import java.util.List; + +public class InvidiousSearchExtractor extends SearchExtractor { + + private final String baseUrl; + private JsonArray results; + + public InvidiousSearchExtractor(StreamingService service, SearchQueryHandler linkHandler) { + super(service, linkHandler); + baseUrl = service.getInstance().getUrl(); + } + + @Nonnull + @Override + public String getSearchSuggestion() throws ParsingException { + return ""; + } + + @Override + public boolean isCorrectedSearch() throws ParsingException { + return false; + } + + @Nonnull + @Override + public List getMetaInfo() throws ParsingException { + return Collections.emptyList(); + } + + @Nonnull + @Override + public InfoItemsPage getInitialPage() throws IOException, ExtractionException { + final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId()); + for (Object o : results) { + collectStreamsFrom(collector, (JsonObject) o); + } + + final Page nextPage = new Page(getUrl() + "&page=" + 2); + + return new InfoItemsPage<>(collector, nextPage); + } + + @Override + public InfoItemsPage getPage(Page page) throws IOException, ExtractionException { + return null; + } + + @Override + public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { + final Response response = downloader.get(getUrl()); + results = InvidiousParsingHelper.getValidJsonArrayFromResponse(response, getUrl()); + } + + private void collectStreamsFrom(InfoItemsSearchCollector collector, JsonObject json) { + final String type = json.getString("type"); + + switch (type) { + case "video": + collector.commit(new InvidiousStreamInfoItemExtractor(json, baseUrl)); + break; + case "playlist": + collector.commit(new InvidiousPlaylistInfoItemExtractor(json, baseUrl)); + break; + case "channel": + collector.commit(new InvidiousChannelInfoItemExtractor(json, baseUrl)); + break; + } + } + +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousStreamExtractor.java new file mode 100644 index 0000000000..01d6a821b8 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousStreamExtractor.java @@ -0,0 +1,394 @@ +package org.schabi.newpipe.extractor.services.youtube.invidious.extractors; + +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.MediaFormat; +import org.schabi.newpipe.extractor.MetaInfo; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.downloader.Response; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.LinkHandler; +import org.schabi.newpipe.extractor.localization.DateWrapper; +import org.schabi.newpipe.extractor.services.youtube.ItagItem; +import org.schabi.newpipe.extractor.services.youtube.invidious.InvidiousParsingHelper; +import org.schabi.newpipe.extractor.stream.*; +import org.schabi.newpipe.extractor.utils.JsonUtils; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.io.IOException; +import java.util.*; + +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl; +import static org.schabi.newpipe.extractor.utils.Utils.isBlank; + +/* + * Copyright (C) 2020 Team NewPipe + * InvidiousStreamExtractor.java is part of NewPipe Extractor. + * + * NewPipe Extractor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe Extractor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe Extractor. If not, see . + */ + +public class InvidiousStreamExtractor extends StreamExtractor { + + private JsonObject json; + private String baseUrl; + + public InvidiousStreamExtractor(StreamingService service, LinkHandler linkHandler) { + super(service, linkHandler); + baseUrl = service.getInstance().getUrl(); + } + + @Nullable + @Override + public String getTextualUploadDate() { + return json.getString("publishedText"); + // Depends on instance localization + } + + @Nullable + @Override + public DateWrapper getUploadDate() { + return InvidiousParsingHelper.getUploadDateFromEpochTime(json.getNumber("published").longValue()); + } + + @Nonnull + @Override + public String getThumbnailUrl() { + final JsonArray thumbnail = json.getArray("authorThumbnails"); + final String url = InvidiousParsingHelper.getThumbnailUrl(thumbnail); + return fixThumbnailUrl(url); + } + + @Nonnull + @Override + public Description getDescription() { + final String descriptionHtml = json.getString("descriptionHtml"); + if (!isBlank(descriptionHtml) || descriptionHtml.equals("

")) { + return new Description(descriptionHtml, Description.HTML); + } + + return new Description(json.getString("description"), Description.PLAIN_TEXT); + } + + @Override + public int getAgeLimit() { + final boolean isFamilyFriendly = json.getBoolean("isFamilyFriendly"); + return isFamilyFriendly ? NO_AGE_LIMIT : 18; + } + + @Override + public long getLength() { + return json.getNumber("lengthSeconds").longValue(); + } + + @Override + public long getTimeStamp() throws ParsingException { + return getTimestampSeconds("((#|&|\\?)t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)"); + // from YouTubeStreamExtractor + } + + @Override + public long getViewCount() { + return json.getNumber("viewCount").longValue(); + } + + @Override + public long getLikeCount() { + return json.getNumber("likeCount").longValue(); + } + + @Override + public long getDislikeCount() { + return json.getNumber("dislikeCount").longValue(); + } + + @Nonnull + @Override + public String getUploaderUrl() { + return baseUrl + json.getString("authorUrl"); + } + + @Nonnull + @Override + public String getUploaderName() { + return json.getString("author"); + } + + @Override + public boolean isUploaderVerified() throws ParsingException { + return false; + } + + @Nonnull + @Override + public String getUploaderAvatarUrl() { + final JsonArray avatars = json.getArray("authorThumbnails"); + final String url = InvidiousParsingHelper.getThumbnailUrl(avatars); + return fixThumbnailUrl(url); + } + + @Nonnull + @Override + public String getSubChannelUrl() { + return ""; + } + + @Nonnull + @Override + public String getSubChannelName() { + return ""; + } + + @Nonnull + @Override + public String getSubChannelAvatarUrl() { + return ""; + } + + @Nonnull + @Override + public String getDashMpdUrl() { + return baseUrl + json.getString("dashUrl"); + } + + @Nonnull + @Override + public String getHlsUrl() { + final String hlsUrl = json.getString("hlsUrl"); + return hlsUrl != null ? hlsUrl : ""; + } + + @Override + public List getAudioStreams() throws ExtractionException { + List audioStreams = new ArrayList<>(); + try { + for (Map.Entry entry : getItags("adaptiveFormats", ItagItem.ItagType.AUDIO).entrySet()) { + ItagItem itag = entry.getValue(); + + AudioStream audioStream = new AudioStream(entry.getKey(), itag); + if (!Stream.containSimilarStream(audioStream, audioStreams)) { + audioStreams.add(audioStream); + } + } + } catch (Exception e) { + throw new ParsingException("Could not get audio streams", e); + } + + return audioStreams; + } + + @Override + public List getVideoStreams() throws ExtractionException { + List videoStreams = new ArrayList<>(); + try { + for (Map.Entry entry : getItags("formatStreams", ItagItem.ItagType.VIDEO).entrySet()) { + ItagItem itag = entry.getValue(); + + VideoStream videoStream = new VideoStream(entry.getKey(), false, itag); + if (!Stream.containSimilarStream(videoStream, videoStreams)) { + videoStreams.add(videoStream); + } + } + } catch (Exception e) { + throw new ParsingException("Could not get video streams", e); + } + + return videoStreams; + } + + @Override + public List getVideoOnlyStreams() throws ExtractionException { + List videoOnlyStreams = new ArrayList<>(); + try { + for (Map.Entry entry : getItags("adaptiveFormats", ItagItem.ItagType.VIDEO_ONLY).entrySet()) { + ItagItem itag = entry.getValue(); + + VideoStream videoStream = new VideoStream(entry.getKey(), true, itag); + if (!Stream.containSimilarStream(videoStream, videoOnlyStreams)) { + videoOnlyStreams.add(videoStream); + } + } + } catch (Exception e) { + throw new ParsingException("Could not get video only streams", e); + } + + return videoOnlyStreams; + } + + @Nonnull + @Override + public List getSubtitlesDefault() { + return getSubtitles(MediaFormat.VTT); + } + + @Nonnull + @Override + public List getSubtitles(MediaFormat format) { + final JsonArray captions = json.getArray("captions"); + List subtitles = new ArrayList<>(captions.size()); + for (Object o : captions) { + final JsonObject obj = (JsonObject) o; + final String languageCode = obj.getString("languageCode"); + subtitles.add(new SubtitlesStream(format, languageCode, baseUrl + obj.getString("url"), languageCode.contains("(auto-generated)"))); + } + + return subtitles; + } + + @Override + public StreamType getStreamType() { + if (json.getBoolean("liveNow")) { + return StreamType.LIVE_STREAM; + } else { + return StreamType.VIDEO_STREAM; + } + } + + @Override + public StreamInfoItemsCollector getRelatedStreams() throws ExtractionException { + try { + final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + final JsonArray relatedStreams = json.getArray("recommendedVideos"); + for (Object o : relatedStreams) { + collector.commit(new InvidiousStreamInfoItemExtractor((JsonObject) o, baseUrl)); + } + return collector; + } catch (Exception e) { + throw new ParsingException("Could not get related videos", e); + } + } + + @Override + public String getErrorMessage() { + return null; + } + + @Nonnull + @Override + public String getHost() { + return ""; + } + + @Nonnull + @Override + public String getPrivacy() { + return json.getBoolean("isListed") ? "Public" : "Unlisted"; + } + + @Nonnull + @Override + public String getCategory() { + return json.getString("genre"); + } + + @Nonnull + @Override + public String getLicence() { + return ""; + } + + @Nullable + @Override + public Locale getLanguageInfo() { + return null; + } + + @Nonnull + @Override + public List getTags() { + return JsonUtils.getListStringFromJsonArray(json.getArray("keywords")); + } + + @Nonnull + @Override + public String getSupportInfo() { + return ""; + } + + @Nonnull + @Override + public List getStreamSegments() throws ParsingException { + return Collections.emptyList(); + } + + @Nonnull + @Override + public List getMetaInfo() throws ParsingException { + return Collections.emptyList(); + } + + @Override + public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { + final String apiUrl = baseUrl + "/api/v1/videos/" + getId() + + "?region=" + getExtractorContentCountry().getCountryCode(); + + final Response response = downloader.get(apiUrl); + + json = InvidiousParsingHelper.getValidJsonObjectFromResponse(response, apiUrl); + } + + @Nonnull + @Override + public String getName() throws ParsingException { + return json.getString("title"); + } + + private Map getItags(final String streamingDataKey, ItagItem.ItagType itagTypeWanted) throws ParsingException { + final JsonArray formats = json.getArray(streamingDataKey); + + Map urlAndItags = new LinkedHashMap<>(); + for (Object o : formats) { + JsonObject formatData = (JsonObject) o; + int itag = Integer.parseInt(formatData.getString("itag")); + + if (ItagItem.isSupported(itag)) { + ItagItem itagItem = ItagItem.getItag(itag); + if (itagItem.itagType == itagTypeWanted) { + urlAndItags.put(formatData.getString("url"), itagItem); + } + } + } + + return urlAndItags; + } + + @Nonnull + @Override + public List getFrames() throws ExtractionException { + final JsonArray storyboards = json.getArray("storyboards"); + List frames = new ArrayList<>(storyboards.size()); + for (Object o : storyboards) { + final JsonObject storyboard = (JsonObject) o; + int durationPerFrames = storyboard.getInt("interval"); + int width = storyboard.getInt("width"); + int height = storyboard.getInt("height"); + int totalCount = storyboard.getInt("count"); + int framesPerPageX = storyboard.getInt("storyboardWidth"); + int framesPerPageY = storyboard.getInt("storyboardHeight"); + int storyBoardCount = storyboard.getInt("storyboardCount"); + List urls = new ArrayList<>(storyBoardCount); + String url = storyboard.getString("templateUrl"); + for (int i = 0; i < storyBoardCount; i++) { + urls.add(url.replace("M$M", "M" + i)); + } + if (durationPerFrames != 0) { + frames.add(new Frameset(urls, width, height, totalCount, durationPerFrames, framesPerPageX, framesPerPageY)); + } + } + return frames; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousStreamInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousStreamInfoItemExtractor.java new file mode 100644 index 0000000000..39ced1bdd9 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousStreamInfoItemExtractor.java @@ -0,0 +1,120 @@ +package org.schabi.newpipe.extractor.services.youtube.invidious.extractors; + +import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.localization.DateWrapper; +import org.schabi.newpipe.extractor.services.youtube.invidious.InvidiousParsingHelper; +import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor; +import org.schabi.newpipe.extractor.stream.StreamType; + +import javax.annotation.Nullable; + +import static org.schabi.newpipe.extractor.services.youtube.invidious.InvidiousParsingHelper.getUploadDateFromEpochTime; + +/* + * Copyright (C) 2020 Team NewPipe + * InvidiousStreamInfoItemExtractor.java is part of NewPipe Extractor. + * + * NewPipe Extractor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe Extractor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe Extractor. If not, see . + */ + +public class InvidiousStreamInfoItemExtractor implements StreamInfoItemExtractor { + + private final JsonObject json; + private final String baseUrl; + + public InvidiousStreamInfoItemExtractor(JsonObject json, String baseUrl) { + this.json = json; + this.baseUrl = baseUrl; + } + + @Override + public StreamType getStreamType() { + if (json.getBoolean("liveNow")) { + return StreamType.LIVE_STREAM; + } else { + return StreamType.VIDEO_STREAM; + } + } + + @Override + public boolean isAd() { + return json.getBoolean("premium") /*|| json.getBoolean("paid")*/; // not sure about this one + } + + @Override + public long getDuration() { + final Number number = json.getNumber("lengthSeconds"); + return number == null ? -1 : number.longValue(); + } + + @Override + public long getViewCount() { + return json.getLong("viewCount", -1); + } + + @Override + public String getUploaderName() { + return json.getString("author"); + } + + @Override + public String getUploaderUrl() { + final String url = json.getString("authorUrl"); + return url != null ? baseUrl + url : baseUrl + "/channel/" + json.getString("authorId"); + } + + @Override + public boolean isUploaderVerified() throws ParsingException { + return false; + } + + @Nullable + @Override + public String getTextualUploadDate() { + return json.getString("publishedText"); + } + + @Nullable + @Override + public DateWrapper getUploadDate() { + final Number epochTime = json.getNumber("published"); + if (epochTime != null) { + return getUploadDateFromEpochTime(epochTime.longValue()); + } + + // maybe use getTextualUploadDate() BUT is unstable because it depends on instance localization + // (or configuration? I mean servers' OS language. That's something to investigate). + // then it won't always be English, and there is no way to know the language from the API + // therefore we should check if the string contains "ago" -> english + + return null; + } + + @Override + public String getName() { + return json.getString("title"); + } + + @Override + public String getUrl() throws ParsingException { + return baseUrl + "/watch?v=" + json.getString("videoId"); + } + + @Override + public String getThumbnailUrl() { + return InvidiousParsingHelper.getThumbnailUrl(json.getArray("videoThumbnails")); + } + +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousSuggestionExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousSuggestionExtractor.java new file mode 100644 index 0000000000..449f5210fd --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousSuggestionExtractor.java @@ -0,0 +1,36 @@ +package org.schabi.newpipe.extractor.services.youtube.invidious.extractors; + +import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.downloader.Response; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.services.youtube.invidious.InvidiousParsingHelper; +import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor; + +import java.io.IOException; +import java.util.List; + +import static org.schabi.newpipe.extractor.utils.JsonUtils.getListStringFromJsonArray; + +public class InvidiousSuggestionExtractor extends SuggestionExtractor { + + private final String baseUrl; + + public InvidiousSuggestionExtractor(StreamingService service) { + super(service); + this.baseUrl = service.getInstance().getUrl(); + } + + @Override + public List suggestionList(String query) throws IOException, ExtractionException { + final String apiUrl = baseUrl + "/api/v1/search/suggestions?q=" + query; + final Downloader dl = NewPipe.getDownloader(); + final Response response = dl.get(apiUrl); + + final JsonObject json = InvidiousParsingHelper.getValidJsonObjectFromResponse(response, apiUrl); + + return getListStringFromJsonArray(json.getArray("suggestions")); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousTrendingExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousTrendingExtractor.java new file mode 100644 index 0000000000..7cc2fb4164 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/extractors/InvidiousTrendingExtractor.java @@ -0,0 +1,59 @@ +package org.schabi.newpipe.extractor.services.youtube.invidious.extractors; + +import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.Page; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.downloader.Response; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.kiosk.KioskExtractor; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.services.youtube.invidious.InvidiousParsingHelper; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; + +import javax.annotation.Nonnull; +import java.io.IOException; + +public class InvidiousTrendingExtractor extends KioskExtractor { + + private JsonArray videos; + private final String baseUrl; + + public InvidiousTrendingExtractor(StreamingService streamingService, ListLinkHandler linkHandler, String kioskId) { + super(streamingService, linkHandler, kioskId); + baseUrl = streamingService.getBaseUrl(); + } + + @Override + public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { + final String apiUrl = baseUrl + "/api/v1/trending?region=" + getExtractorContentCountry(); + final Response response = downloader.get(apiUrl); + videos = InvidiousParsingHelper.getValidJsonArrayFromResponse(response, apiUrl); + } + + @Nonnull + @Override + public InfoItemsPage getInitialPage() throws IOException, ExtractionException { + final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + for (Object o : videos) { + collector.commit(new InvidiousStreamInfoItemExtractor((JsonObject) o, baseUrl)); + } + + return new InfoItemsPage<>(collector, null); + } + + @Override + public InfoItemsPage getPage(Page page) throws IOException, ExtractionException { + return InfoItemsPage.emptyPage(); + } + + @Nonnull + @Override + public String getName() throws ParsingException { + return "Trending"; + } + +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/linkhandler/InvidiousSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/linkhandler/InvidiousSearchQueryHandlerFactory.java new file mode 100644 index 0000000000..83f01c4762 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/invidious/linkhandler/InvidiousSearchQueryHandlerFactory.java @@ -0,0 +1,57 @@ +package org.schabi.newpipe.extractor.services.youtube.invidious.linkhandler; + +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory; + +import java.util.List; + +public class InvidiousSearchQueryHandlerFactory extends SearchQueryHandlerFactory { + + public static final String ALL = "all"; + public static final String VIDEOS = "videos"; + public static final String CHANNELS = "channels"; + public static final String PLAYLISTS = "playlists"; + + private static String baseUrl; + + private InvidiousSearchQueryHandlerFactory(final String baseUrl) { + InvidiousSearchQueryHandlerFactory.baseUrl = baseUrl; + } + + public static InvidiousSearchQueryHandlerFactory getInstance(final String baseUrl) { + return new InvidiousSearchQueryHandlerFactory(baseUrl); + } + + @Override + public String getUrl(String query, List contentFilter, String sortFilter) throws ParsingException { + String url = baseUrl + "/api/v1/search?q=" + query; + + if (contentFilter.size() > 0) { + switch (contentFilter.get(0)) { + case VIDEOS: + return url; // + "&type=video" it's the default type provided by Invidious + case CHANNELS: + return url + "&type=channel"; + case PLAYLISTS: + return url + "&type=playlist"; + case ALL: + default: + break; + } + } + + return url + "&type=all"; + } + + + @Override + public String[] getAvailableContentFilter() { + return new String[]{ + ALL, + VIDEOS, + CHANNELS, + PLAYLISTS + }; + } + +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java index 2dc8fc427c..78406430b8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java @@ -1,6 +1,5 @@ package org.schabi.newpipe.extractor.services.youtube.linkHandler; -import java.util.regex.Pattern; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; @@ -8,6 +7,7 @@ import java.net.URL; import java.util.List; +import java.util.regex.Pattern; /* * Created by Christian Schabesberger on 25.07.16. @@ -34,7 +34,7 @@ public class YoutubeChannelLinkHandlerFactory extends ListLinkHandlerFactory { private static final YoutubeChannelLinkHandlerFactory instance = new YoutubeChannelLinkHandlerFactory(); private static final Pattern excludedSegments = - Pattern.compile("playlist|watch|attribution_link|watch_popup|embed|feed|select_site"); + Pattern.compile("playlist|watch|attribution_link|watch_popup|embed|feed|select_site"); public static YoutubeChannelLinkHandlerFactory getInstance() { return instance; @@ -43,7 +43,7 @@ public static YoutubeChannelLinkHandlerFactory getInstance() { /** * Returns URL to channel from an ID * - * @param id Channel ID including e.g. 'channel/' + * @param id Channel ID including e.g. 'channel/' * @param contentFilters * @param searchFilter * @return URL to channel @@ -52,7 +52,7 @@ public static YoutubeChannelLinkHandlerFactory getInstance() { public String getUrl(String id, List contentFilters, String searchFilter) { return "https://www.youtube.com/" + id; } - + /** * Returns true if path conform to * custom short channel URLs like youtube.com/yourcustomname @@ -71,7 +71,8 @@ public String getId(String url) throws ParsingException { String path = urlObj.getPath(); if (!Utils.isHTTP(urlObj) || !(YoutubeParsingHelper.isYoutubeURL(urlObj) || - YoutubeParsingHelper.isInvidioURL(urlObj) || YoutubeParsingHelper.isHooktubeURL(urlObj))) { + YoutubeParsingHelper.isInvidiousRedirectUrl(urlObj) || YoutubeParsingHelper.isHooktubeURL(urlObj) + || YoutubeParsingHelper.isInvidiousURL(urlObj))) { throw new ParsingException("the URL given is not a Youtube-URL"); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java index 9ed2ae7752..b44fc47dd4 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java @@ -1,8 +1,5 @@ package org.schabi.newpipe.extractor.services.youtube.linkHandler; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.List; import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.LinkHandler; @@ -11,6 +8,10 @@ import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; import org.schabi.newpipe.extractor.utils.Utils; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.List; + public class YoutubePlaylistLinkHandlerFactory extends ListLinkHandlerFactory { private static final YoutubePlaylistLinkHandlerFactory INSTANCE = @@ -32,7 +33,8 @@ public String getId(final String url) throws ParsingException { final URL urlObj = Utils.stringToURL(url); if (!Utils.isHTTP(urlObj) || !(YoutubeParsingHelper.isYoutubeURL(urlObj) - || YoutubeParsingHelper.isInvidioURL(urlObj))) { + || YoutubeParsingHelper.isInvidiousRedirectUrl(urlObj) + || YoutubeParsingHelper.isInvidiousURL(urlObj))) { throw new ParsingException("the url given is not a YouTube-URL"); } @@ -91,14 +93,14 @@ public ListLinkHandler fromUrl(final String url) throws ParsingException { videoID = YoutubeParsingHelper.extractVideoIdFromMixId(listID); } final String newUrl = "https://www.youtube.com/watch?v=" + videoID - + "&list=" + listID; + + "&list=" + listID; return new ListLinkHandler(new LinkHandler(url, newUrl, listID), getContentFilter(url), getSortFilter(url)); } } catch (MalformedURLException exception) { throw new ParsingException("Error could not parse URL: " + exception.getMessage(), - exception); + exception); } return super.fromUrl(url); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java index bff9d82777..72c889af30 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeStreamLinkHandlerFactory.java @@ -1,5 +1,6 @@ package org.schabi.newpipe.extractor.services.youtube.linkHandler; +import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.FoundAdException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; @@ -16,6 +17,10 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import static org.schabi.newpipe.extractor.ServiceList.Invidious; +import static org.schabi.newpipe.extractor.ServiceList.YouTube; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*; + /* * Created by Christian Schabesberger on 02.02.16. * @@ -69,7 +74,13 @@ private static String assertIsId(@Nullable final String id) throws ParsingExcept @Override public String getUrl(String id) { - return "https://www.youtube.com/watch?v=" + id; + final String baseUrl; + if (NewPipe.getUseInvidiousForYoutube()) { + baseUrl = Invidious.getBaseUrl(); + } else { + baseUrl = YouTube.getBaseUrl(); + } + return baseUrl + "/watch?v=" + id; } @Override @@ -108,9 +119,10 @@ public String getId(String urlString) throws ParsingException, IllegalArgumentEx path = path.substring(1); } - if (!Utils.isHTTP(url) || !(YoutubeParsingHelper.isYoutubeURL(url) || - YoutubeParsingHelper.isYoutubeServiceURL(url) || YoutubeParsingHelper.isHooktubeURL(url) || - YoutubeParsingHelper.isInvidioURL(url))) { + final boolean isInvidiousUrl = YoutubeParsingHelper.isInvidiousURL(url); // save it to call it only once + + if (!Utils.isHTTP(url) || !(isInvidiousUrl || isYoutubeURL(url) || isYoutubeServiceUrl(url) + || isInvidiousRedirectUrl(url) || isHooktubeURL(url))) { if (host.equalsIgnoreCase("googleads.g.doubleclick.net")) { throw new FoundAdException("Error found ad: " + urlString); } @@ -124,87 +136,56 @@ public String getId(String urlString) throws ParsingException, IllegalArgumentEx // using uppercase instead of lowercase, because toLowercase replaces some unicode characters // with their lowercase ASCII equivalent. Using toLowercase could result in faultily matching unicode urls. - switch (host.toUpperCase()) { - case "WWW.YOUTUBE-NOCOOKIE.COM": { - if (path.startsWith("embed/")) { - String id = path.substring(6); // embed/ - - return assertIsId(id); + final String hostUpperCase = host.toUpperCase(); + + if (hostUpperCase.equals("WWW.YOUTUBE-NOCOOKIE.COM") && path.startsWith("embed/")) { + String id = path.substring(6); // embed/ + return assertIsId(id); + } else if (hostUpperCase.equals("YOUTUBE.COM") || hostUpperCase.equals("WWW.YOUTUBE.COM") + || hostUpperCase.equals("M.YOUTUBE.COM") || hostUpperCase.equals("MUSIC.YOUTUBE.COM")) { + if (path.equals("attribution_link")) { + String uQueryValue = Utils.getQueryValue(url, "u"); + + URL decodedURL; + try { + decodedURL = Utils.stringToURL("http://www.youtube.com" + uQueryValue); + } catch (MalformedURLException e) { + throw new ParsingException("Error no suitable url: " + urlString); } - break; + String viewQueryValue = Utils.getQueryValue(decodedURL, "v"); + return assertIsId(viewQueryValue); } - case "YOUTUBE.COM": - case "WWW.YOUTUBE.COM": - case "M.YOUTUBE.COM": - case "MUSIC.YOUTUBE.COM": { - if (path.equals("attribution_link")) { - String uQueryValue = Utils.getQueryValue(url, "u"); - - URL decodedURL; - try { - decodedURL = Utils.stringToURL("http://www.youtube.com" + uQueryValue); - } catch (MalformedURLException e) { - throw new ParsingException("Error no suitable url: " + urlString); - } - - String viewQueryValue = Utils.getQueryValue(decodedURL, "v"); - return assertIsId(viewQueryValue); - } - - String maybeId = getIdFromSubpathsInPath(path); - if (maybeId != null) return maybeId; + String maybeId = getIdFromSubpathsInPath(path); + if (maybeId != null) return maybeId; - String viewQueryValue = Utils.getQueryValue(url, "v"); + String viewQueryValue = Utils.getQueryValue(url, "v"); + return assertIsId(viewQueryValue); + } else if (hostUpperCase.equals("YOUTU.BE")) { + String viewQueryValue = Utils.getQueryValue(url, "v"); + if (viewQueryValue != null) { return assertIsId(viewQueryValue); } - case "YOUTU.BE": { + return assertIsId(path); + } else if (isInvidiousUrl || hostUpperCase.equals("HOOKTUBE.COM") || isInvidiousRedirectUrl(url)) { + if (path.equals("watch")) { String viewQueryValue = Utils.getQueryValue(url, "v"); if (viewQueryValue != null) { return assertIsId(viewQueryValue); } - - return assertIsId(path); } - case "HOOKTUBE.COM": - case "INVIDIO.US": - case "DEV.INVIDIO.US": - case "WWW.INVIDIO.US": - case "REDIRECT.INVIDIOUS.IO": - case "INVIDIOUS.SNOPYTA.ORG": - case "YEWTU.BE": - case "TUBE.CONNECT.CAFE": - case "INVIDIOUS.ZAPASHCANON.FR": - case "INVIDIOUS.KAVIN.ROCKS": - case "INVIDIOUS.TUBE": - case "INVIDIOUS.SITE": - case "INVIDIOUS.XYZ": - case "VID.MINT.LGBT": - case "INVIDIOU.SITE": - case "INVIDIOUS.FDN.FR": - case "INVIDIOUS.048596.XYZ": - case "INVIDIOUS.ZEE.LI": - case "VID.PUFFYAN.US": - case "YTPRIVATE.COM": { // code-block for hooktube.com and Invidious instances - if (path.equals("watch")) { - String viewQueryValue = Utils.getQueryValue(url, "v"); - if (viewQueryValue != null) { - return assertIsId(viewQueryValue); - } - } - String maybeId = getIdFromSubpathsInPath(path); - if (maybeId != null) return maybeId; + String maybeId = getIdFromSubpathsInPath(path); + if (maybeId != null) return maybeId; - String viewQueryValue = Utils.getQueryValue(url, "v"); - if (viewQueryValue != null) { - return assertIsId(viewQueryValue); - } - - return assertIsId(path); + String viewQueryValue = Utils.getQueryValue(url, "v"); + if (viewQueryValue != null) { + return assertIsId(viewQueryValue); } + + return assertIsId(path); } throw new ParsingException("Error no suitable url: " + urlString); @@ -231,4 +212,5 @@ private String getIdFromSubpathsInPath(String path) throws ParsingException { } return null; } + } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeTrendingLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeTrendingLinkHandlerFactory.java index 6d6db7424a..d8150ca141 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeTrendingLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeTrendingLinkHandlerFactory.java @@ -28,6 +28,8 @@ import java.net.URL; import java.util.List; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*; + public class YoutubeTrendingLinkHandlerFactory extends ListLinkHandlerFactory { public String getUrl(String id, List contentFilters, String sortFilter) { @@ -49,6 +51,8 @@ public boolean onAcceptUrl(final String url) { } String urlPath = urlObj.getPath(); - return Utils.isHTTP(urlObj) && (YoutubeParsingHelper.isYoutubeURL(urlObj) || YoutubeParsingHelper.isInvidioURL(urlObj)) && urlPath.equals("/feed/trending"); + return Utils.isHTTP(urlObj) + && (isYoutubeURL(urlObj) || isInvidiousRedirectUrl(urlObj) || isInvidiousURL(urlObj)) + && urlPath.equals("/feed/trending"); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java index ad132b5ac8..d6810914c8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java @@ -99,4 +99,14 @@ private static JsonObject getObject(@Nonnull JsonObject object, @Nonnull List getListStringFromJsonArray(@Nonnull JsonArray array) { + List strings = new ArrayList<>(array.size()); + for (Object obj : array) { + if (obj instanceof String) { + strings.add((String) obj); + } + } + return strings; + } + } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java index 721bc96c3c..c8c559f4e6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java @@ -183,7 +183,7 @@ public static String removeUTF8BOM(String s) { public static String getBaseUrl(final String url) throws ParsingException { try { final URL uri = stringToURL(url); - return uri.getProtocol() + "://" + uri.getAuthority(); + return getBaseUrl(uri); } catch (final MalformedURLException e) { final String message = e.getMessage(); if (message.startsWith("unknown protocol: ")) { @@ -195,6 +195,10 @@ public static String getBaseUrl(final String url) throws ParsingException { } } + public static String getBaseUrl(final URL url) { + return url.getProtocol() + "://" + url.getAuthority(); + } + /** * If the provided url is a Google search redirect, then the actual url is extracted from the * {@code url=} query value and returned, otherwise the original url is returned. diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/NewPipeTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/NewPipeTest.java index 5dbc431747..7981adf159 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/NewPipeTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/NewPipeTest.java @@ -1,15 +1,23 @@ package org.schabi.newpipe.extractor; +import org.junit.BeforeClass; import org.junit.Test; +import org.schabi.newpipe.downloader.DownloaderTestImpl; import java.util.HashSet; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import static org.schabi.newpipe.extractor.NewPipe.getServiceByUrl; -import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; -import static org.schabi.newpipe.extractor.ServiceList.YouTube; +import static org.schabi.newpipe.extractor.ServiceList.*; public class NewPipeTest { + + @BeforeClass + public static void setUp() { + NewPipe.init(DownloaderTestImpl.getInstance()); + } + @Test public void getAllServicesTest() throws Exception { assertEquals(NewPipe.getServices().size(), ServiceList.all().size()); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YouTubeCommentsLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YouTubeCommentsLinkHandlerFactoryTest.java index afc538e1ea..4b33012463 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YouTubeCommentsLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YouTubeCommentsLinkHandlerFactoryTest.java @@ -5,6 +5,7 @@ import org.schabi.newpipe.downloader.DownloaderTestImpl; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeCommentsLinkHandlerFactory; import static org.junit.Assert.assertEquals; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelperTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelperTest.java index 39af59e146..ddfd0afa16 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelperTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelperTest.java @@ -6,11 +6,13 @@ import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.*; public class YoutubeParsingHelperTest { @@ -48,4 +50,14 @@ public void testConvertFromGoogleCacheUrl() throws ParsingException { assertEquals("https://www.infektionsschutz.de/coronavirus-sars-cov-2.html", YoutubeParsingHelper.extractCachedUrlIfNeeded("https://www.infektionsschutz.de/coronavirus-sars-cov-2.html")); } + + @Test + public void testIsYoutubeUrl() throws MalformedURLException { + final URL url = Utils.stringToURL("https://youtube.com"); + assertTrue(YoutubeParsingHelper.isYoutubeURL(url)); + assertFalse(YoutubeParsingHelper.isInvidiousRedirectUrl(url)); + assertFalse(YoutubeParsingHelper.isInvidiousURL(url)); + assertFalse(YoutubeParsingHelper.isHooktubeURL(url)); + } + } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistLinkHandlerFactoryTest.java index 3a02983460..db973d1b55 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistLinkHandlerFactoryTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistLinkHandlerFactoryTest.java @@ -70,7 +70,7 @@ public void testDeniesInvalidYtUrl() throws ParsingException { } @Test - public void testAcceptInvidioUrl() throws ParsingException { + public void testAcceptInvidiousUrl() throws ParsingException { assertTrue(linkHandler.acceptUrl("https://www.invidio.us/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC")); assertTrue(linkHandler.acceptUrl("https://www.invidio.us/playlist?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV")); assertTrue(linkHandler.acceptUrl("https://WWW.invidio.us/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dCI")); @@ -83,7 +83,7 @@ public void testAcceptInvidioUrl() throws ParsingException { } @Test - public void testDeniesInvalidInvidioUrl() throws ParsingException { + public void testDeniesInvalidInvidiousUrl() throws ParsingException { assertFalse(linkHandler.acceptUrl("https://invidio.us/feed/trending?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV")); assertFalse(linkHandler.acceptUrl("https://invidio.us/feed/subscriptions?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV")); assertFalse(linkHandler.acceptUrl("ftp:/invidio.us/feed/trending?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV")); @@ -93,7 +93,7 @@ public void testDeniesInvalidInvidioUrl() throws ParsingException { } @Test - public void testGetInvidioIdfromUrl() throws ParsingException { + public void testGetInvidiousIdfromUrl() throws ParsingException { assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("https://www.invidio.us/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC").getId()); assertEquals("PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV", linkHandler.fromUrl("https://www.invidio.us/playlist?list=PLz8YL4HVC87WJQDzVoY943URKQCsHS9XV").getId()); assertEquals("PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC", linkHandler.fromUrl("https://www.invidio.us/playlist?list=PLW5y1tjAOzI3orQNF1yGGVL5x-pR2K1dC&t=100").getId()); @@ -122,6 +122,6 @@ public void fromUrlIsMixPlaylist() throws Exception { final String videoId = "_AzeUSL9lZc"; final String url = "https://www.youtube.com/watch?v=" + videoId + "&list=RD" + videoId; assertEquals(url, - linkHandler.fromUrl("https://www.youtube.com/watch?list=RD" + videoId).getUrl()); + linkHandler.fromUrl("https://www.youtube.com/watch?list=RD" + videoId).getUrl()); } } \ No newline at end of file diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousInstanceTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousInstanceTest.java new file mode 100644 index 0000000000..4bbb851185 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/invidious/InvidiousInstanceTest.java @@ -0,0 +1,36 @@ +package org.schabi.newpipe.extractor.services.youtube.invidious; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.schabi.newpipe.downloader.DownloaderTestImpl; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.InvalidInstanceException; + +import static org.junit.Assert.*; + +public class InvidiousInstanceTest { + + @BeforeClass + public static void setUp() { + NewPipe.init(DownloaderTestImpl.getInstance()); + } + + @Test + public void testInvidious_snopyta_IsValid() { + InvidiousInstance invidious = new InvidiousInstance("https://invidious.snopyta.org"); + assertTrue(invidious.isValid()); + } + + @Test + public void testInvidious_snopyta_GetName() throws InvalidInstanceException { + InvidiousInstance invidious = new InvidiousInstance("https://invidious.snopyta.org"); + invidious.fetchInstanceMetaData(); + assertEquals("invidious", invidious.getName()); + } + + @Test + public void testYoutube_comIsValid() { + InvidiousInstance youtube = new InvidiousInstance("https://youtube.com"); + assertFalse(youtube.isValid()); + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/invidious/stream/InvidiousStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/invidious/stream/InvidiousStreamExtractorDefaultTest.java new file mode 100644 index 0000000000..ae704b9b62 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/invidious/stream/InvidiousStreamExtractorDefaultTest.java @@ -0,0 +1,386 @@ +//package org.schabi.newpipe.extractor.services.youtube.invidious.stream; +// +//import org.junit.BeforeClass; +//import org.junit.Ignore; +//import org.junit.Test; +//import org.schabi.newpipe.downloader.DownloaderFactory; +//import org.schabi.newpipe.extractor.MetaInfo; +//import org.schabi.newpipe.extractor.NewPipe; +//import org.schabi.newpipe.extractor.StreamingService; +//import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; +//import org.schabi.newpipe.extractor.exceptions.ParsingException; +//import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest; +//import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; +//import org.schabi.newpipe.extractor.services.youtube.invidious.InvidiousInstance; +//import org.schabi.newpipe.extractor.stream.Description; +//import org.schabi.newpipe.extractor.stream.StreamExtractor; +//import org.schabi.newpipe.extractor.stream.StreamSegment; +//import org.schabi.newpipe.extractor.stream.StreamType; +// +//import javax.annotation.Nullable; +//import java.io.IOException; +//import java.net.MalformedURLException; +//import java.net.URL; +//import java.util.Arrays; +//import java.util.Collections; +//import java.util.List; +// +//import static org.junit.Assert.assertEquals; +//import static org.junit.Assert.assertNotNull; +//import static org.schabi.newpipe.extractor.ServiceList.Invidious; +//import static org.schabi.newpipe.extractor.ServiceList.YouTube; +//import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING; +// +///* +// * Created by Christian Schabesberger on 30.12.15. +// * +// * Copyright (C) Christian Schabesberger 2015 +// * YoutubeVideoExtractorDefault.java is part of NewPipe. +// * +// * NewPipe is free software: you can redistribute it and/or modify +// * it under the terms of the GNU General Public License as published by +// * the Free Software Foundation, either version 3 of the License, or +// * (at your option) any later version. +// * +// * NewPipe is distributed in the hope that it will be useful, +// * but WITHOUT ANY WARRANTY; without even the implied warranty of +// * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// * GNU General Public License for more details. +// * +// * You should have received a copy of the GNU General Public License +// * along with NewPipe. If not, see . +// */ +//public class InvidiousStreamExtractorDefaultTest { +// private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/stream/"; +// static final String INSTANCE_URL = InvidiousInstance.getDefaultInstance().getUrl(); +// static final String BASE_URL = INSTANCE_URL + "/watch?v="; +// +// public static class NotAvailable { +// @BeforeClass +// public static void setUp() throws IOException { +// YoutubeParsingHelper.resetClientVersionAndKey(); +// NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "notAvailable")); +// NewPipe.setUseInvidiousForYoutube(true); +// } +// +// @Test(expected = ContentNotAvailableException.class) +// public void nonExistentFetch() throws Exception { +// final StreamExtractor extractor = +// YouTube.getStreamExtractor(BASE_URL + "don-t-exist"); +// extractor.fetchPage(); +// } +// +// @Test(expected = ParsingException.class) +// public void invalidId() throws Exception { +// final StreamExtractor extractor = +// YouTube.getStreamExtractor(BASE_URL + "INVALID_ID_INVALID_ID"); +// extractor.fetchPage(); +// } +// } +// +// public static class DescriptionTestPewdiepie extends DefaultStreamExtractorTest { +// private static final String ID = "7PIMiDcwNvc"; +// private static final int TIMESTAMP = 17; +// private static final String URL = BASE_URL + ID + "&t=" + TIMESTAMP; +// private static StreamExtractor extractor; +// +// @BeforeClass +// public static void setUp() throws Exception { +// YoutubeParsingHelper.resetClientVersionAndKey(); +// NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "pewdiwpie")); +// NewPipe.setUseInvidiousForYoutube(true); +// extractor = Invidious.getStreamExtractor(URL); +// extractor.fetchPage(); +// } +// +// // @formatter:off +// @Override public StreamExtractor extractor() { return extractor; } +// @Override public StreamingService expectedService() { return YouTube; } +// @Override public String expectedName() { return "Marzia & Felix - Wedding 19.08.2019"; } +// @Override public String expectedId() { return ID; } +// @Override public String expectedUrlContains() { return BASE_URL + ID; } +// @Override public String expectedOriginalUrlContains() { return URL; } +// +// @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; } +// @Override public String expectedUploaderName() { return "PewDiePie"; } +// @Override public String expectedUploaderUrl() { return BASE_URL + "/channel/UC-lHJZR3Gqxm24_Vd_AJ5Yw"; } +// @Override public boolean expectedUploaderVerified() { return false; } // Invidious does not extract verified status +// @Override public long expectedLength() { return 382; } +// @Override public long expectedTimestamp() { return TIMESTAMP; } +// @Override public long expectedViewCountAtLeast() { return 26682500; } +// @Nullable @Override public String expectedUploadDate() { return "2019-08-24 07:00:00.000"; } +// @Nullable @Override public String expectedTextualUploadDate() { return "1 year ago"; } +// @Override public long expectedLikeCountAtLeast() { return 5212900; } +// @Override public long expectedDislikeCountAtLeast() { return 30600; } +// @Override public int expectedStreamSegmentsCount() { return 0; } +// @Override public String expectedPrivacy() { return "Public"; } +// @Override public List expectedDescriptionContains() { +// return Arrays.asList("https://www.youtube.com/channel/UC7l23W7gFi4Uho6WSzckZRA", +// "https://www.handcraftpictures.com/"); +// } +// @Override public String expectedCategory() { return "Entertainment"; } +// // @formatter:on +// @Ignore("bug https://github.com/iv-org/invidious/issues/1767") +// @Override +// @Test +// public void testDescription() throws Exception { +// super.testDescription(); +// } +// } +// +// public static class DescriptionTestUnboxing extends DefaultStreamExtractorTest { +// private static final String ID = "cV5TjZCJkuA"; +// private static final String URL = BASE_URL + ID; +// private static StreamExtractor extractor; +// +// @BeforeClass +// public static void setUp() throws Exception { +// YoutubeParsingHelper.resetClientVersionAndKey(); +// NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "unboxing")); +// NewPipe.setUseInvidiousForYoutube(true); +// extractor = Invidious.getStreamExtractor(URL); +// extractor.fetchPage(); +// } +// +// // @formatter:off +// @Override public StreamExtractor extractor() { return extractor; } +// @Override public StreamingService expectedService() { return YouTube; } +// @Override public String expectedName() { return "This Smartphone Changes Everything..."; } +// @Override public String expectedId() { return ID; } +// @Override public String expectedUrlContains() { return URL; } +// @Override public String expectedOriginalUrlContains() { return URL; } +// +// @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; } +// @Override public String expectedUploaderName() { return "Unbox Therapy"; } +// @Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCsTcErHg8oDvUnTzoqsYeNw"; } +// @Override public List expectedDescriptionContains() { +// return Arrays.asList("https://www.youtube.com/watch?v=X7FLCHVXpsA&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34", +// "https://www.youtube.com/watch?v=Lqv6G0pDNnw&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34", +// "https://www.youtube.com/watch?v=XxaRBPyrnBU&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34", +// "https://www.youtube.com/watch?v=U-9tUEOFKNU&list=PL7u4lWXQ3wfI_7PgX0C-VTiwLeu0S4v34"); +// } +// @Override public long expectedLength() { return 434; } +// @Override public long expectedViewCountAtLeast() { return 21229200; } +// @Nullable @Override public String expectedUploadDate() { return "2018-06-19 00:00:00.000"; } +// @Nullable @Override public String expectedTextualUploadDate() { return "2 years ago"; } +// @Override public long expectedLikeCountAtLeast() { return 340100; } +// @Override public long expectedDislikeCountAtLeast() { return 18700; } +// @Override public boolean expectedUploaderVerified() { return true; } +// // @formatter:on +// @Override +// @Test +// @Ignore("TODO fix") +// public void testDescription() throws Exception { +// super.testDescription(); +// } +// +// @Override +// public List expectedTags() { +// return Arrays.asList("2018", "8 plus", "apple", "apple iphone", "apple iphone x", "best", "best android", +// "best smartphone", "cool gadgets", "find", "find x", "find x review", "find x unboxing", "findx", +// "galaxy s9", "galaxy s9+", "hands on", "iphone 8", "iphone 8 plus", "iphone x", "new iphone", "nex", +// "oneplus 6", "oppo", "oppo find x", "oppo find x hands on", "oppo find x review", +// "oppo find x unboxing", "oppo findx", "pixel 2 xl", "review", "samsung", "samsung galaxy", +// "samsung galaxy s9", "smartphone", "unbox therapy", "unboxing", "vivo", "vivo apex", "vivo nex"); +// } +// } +// +// @Ignore("TODO fix") +// public static class RatingsDisabledTest extends DefaultStreamExtractorTest { +// private static final String ID = "HRKu0cvrr_o"; +// private static final int TIMESTAMP = 17; +// private static final String URL = BASE_URL + ID + "&t=" + TIMESTAMP; +// private static StreamExtractor extractor; +// +// @BeforeClass +// public static void setUp() throws Exception { +// YoutubeParsingHelper.resetClientVersionAndKey(); +// NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "ratingsDisabled")); +// extractor = YouTube.getStreamExtractor(URL); +// extractor.fetchPage(); +// } +// +// // @formatter:off +// @Override public StreamExtractor extractor() { return extractor; } +// @Override public StreamingService expectedService() { return YouTube; } +// @Override public String expectedName() { return "AlphaOmegaSin Fanboy Logic: Likes/Dislikes Disabled = Point Invalid Lol wtf?"; } +// @Override public String expectedId() { return ID; } +// @Override public String expectedUrlContains() { return BASE_URL + ID; } +// @Override public String expectedOriginalUrlContains() { return URL; } +// +// @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; } +// @Override public String expectedUploaderName() { return "YouTuber PrinceOfFALLEN"; } +// @Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCQT2yul0lr6Ie9qNQNmw-sg"; } +// @Override public List expectedDescriptionContains() { return Arrays.asList("dislikes", "Alpha", "wrong"); } +// @Override public long expectedLength() { return 84; } +// @Override public long expectedTimestamp() { return TIMESTAMP; } +// @Override public long expectedViewCountAtLeast() { return 190; } +// @Nullable @Override public String expectedUploadDate() { return "2019-01-02 00:00:00.000"; } +// @Nullable @Override public String expectedTextualUploadDate() { return "2019-01-02"; } +// @Override public long expectedLikeCountAtLeast() { return -1; } +// @Override public long expectedDislikeCountAtLeast() { return -1; } +// // @formatter:on +// } +// +// public static class StreamSegmentsTestOstCollection extends DefaultStreamExtractorTest { +// // StreamSegment example with single macro-makers panel +// private static final String ID = "2RYrHwnLHw0"; +// private static final String URL = BASE_URL + ID; +// private static StreamExtractor extractor; +// +// @BeforeClass +// public static void setUp() throws Exception { +// YoutubeParsingHelper.resetClientVersionAndKey(); +// NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "streamSegmentsOstCollection")); +// extractor = YouTube.getStreamExtractor(URL); +// extractor.fetchPage(); +// } +// +// // @formatter:off +// @Override public StreamExtractor extractor() { return extractor; } +// @Override public StreamingService expectedService() { return YouTube; } +// @Override public String expectedName() { return "1 Hour - Most Epic Anime Mix - Battle Anime OST"; } +// @Override public String expectedId() { return ID; } +// @Override public String expectedUrlContains() { return BASE_URL + ID; } +// @Override public String expectedOriginalUrlContains() { return URL; } +// +// @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; } +// @Override public String expectedUploaderName() { return "MathCaires"; } +// @Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UChFoHg6IT18SCqiwCp_KY7Q"; } +// @Override public List expectedDescriptionContains() { +// return Arrays.asList("soundtracks", "9:49", "YouSeeBIGGIRLTT"); +// } +// @Override public long expectedLength() { return 3889; } +// @Override public long expectedViewCountAtLeast() { return 2463261; } +// @Nullable @Override public String expectedUploadDate() { return "2019-06-26 00:00:00.000"; } +// @Nullable @Override public String expectedTextualUploadDate() { return "2019-06-26"; } +// @Override public long expectedLikeCountAtLeast() { return 32100; } +// @Override public long expectedDislikeCountAtLeast() { return 750; } +// @Override public boolean expectedHasSubtitles() { return false; } +// +// @Override public int expectedStreamSegmentsCount() { return 17; } +// @Test +// public void testStreamSegment() throws Exception { +// final StreamSegment segment = extractor.getStreamSegments().get(3); +// assertEquals(589, segment.getStartTimeSeconds()); +// assertEquals("Attack on Titan S2 - YouSeeBIGGIRLTT", segment.getTitle()); +// assertEquals(BASE_URL + ID + "?t=589", segment.getUrl()); +// assertNotNull(segment.getPreviewUrl()); +// } +// // @formatter:on +// } +// +// public static class StreamSegmentsTestMaiLab extends DefaultStreamExtractorTest { +// // StreamSegment example with macro-makers panel and transcription panel +// private static final String ID = "ud9d5cMDP_0"; +// private static final String URL = BASE_URL + ID; +// private static StreamExtractor extractor; +// +// @BeforeClass +// public static void setUp() throws Exception { +// YoutubeParsingHelper.resetClientVersionAndKey(); +// NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "streamSegmentsMaiLab")); +// extractor = YouTube.getStreamExtractor(URL); +// extractor.fetchPage(); +// } +// +// // @formatter:off +// @Override public StreamExtractor extractor() { return extractor; } +// @Override public StreamingService expectedService() { return YouTube; } +// @Override public String expectedName() { return "Vitamin D wissenschaftlich gepr\u00fcft"; } +// @Override public String expectedId() { return ID; } +// @Override public String expectedUrlContains() { return BASE_URL + ID; } +// @Override public String expectedOriginalUrlContains() { return URL; } +// +// @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; } +// @Override public String expectedUploaderName() { return "maiLab"; } +// @Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCyHDQ5C6z1NDmJ4g6SerW8g"; } +// @Override public List expectedDescriptionContains() {return Arrays.asList("Vitamin", "2:44", "Was ist Vitamin D?");} +// @Override public boolean expectedUploaderVerified() { return true; } +// @Override public long expectedLength() { return 1010; } +// @Override public long expectedViewCountAtLeast() { return 815500; } +// @Nullable @Override public String expectedUploadDate() { return "2020-11-18 00:00:00.000"; } +// @Nullable @Override public String expectedTextualUploadDate() { return "2020-11-18"; } +// @Override public long expectedLikeCountAtLeast() { return 48500; } +// @Override public long expectedDislikeCountAtLeast() { return 20000; } +// @Override public boolean expectedHasSubtitles() { return true; } +// @Override public int expectedStreamSegmentsCount() { return 7; } +// // @formatter:on +// +// @Test +// public void testStreamSegment() throws Exception { +// final StreamSegment segment = extractor.getStreamSegments().get(1); +// assertEquals(164, segment.getStartTimeSeconds()); +// assertEquals("Was ist Vitamin D?", segment.getTitle()); +// assertEquals(BASE_URL + ID + "?t=164", segment.getUrl()); +// assertNotNull(segment.getPreviewUrl()); +// } +// +// @Override +// @Test +// @Ignore("encoding problem") +// public void testName() throws Exception { +// super.testName(); +// } +// } +// +// public static class PublicBroadcasterTest extends DefaultStreamExtractorTest { +// private static final String ID = "q6fgbYWsMgw"; +// private static final int TIMESTAMP = 0; +// private static final String URL = BASE_URL + ID; +// private static StreamExtractor extractor; +// +// @BeforeClass +// public static void setUp() throws Exception { +// YoutubeParsingHelper.resetClientVersionAndKey(); +// NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "publicBroadcast")); +// extractor = YouTube.getStreamExtractor(URL); +// extractor.fetchPage(); +// } +// +// // @formatter:off +// @Override public StreamExtractor extractor() { return extractor; } +// @Override public StreamingService expectedService() { return YouTube; } +// @Override public String expectedName() { return "Was verbirgt sich am tiefsten Punkt des Ozeans?"; } +// @Override public String expectedId() { return ID; } +// @Override public String expectedUrlContains() { return BASE_URL + ID; } +// @Override public String expectedOriginalUrlContains() { return URL; } +// +// @Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; } +// @Override public String expectedUploaderName() { return "Dinge Erklärt – Kurzgesagt"; } +// @Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCwRH985XgMYXQ6NxXDo8npw"; } +// @Override public List expectedDescriptionContains() { return Arrays.asList("Lasst uns abtauchen!", "Angebot von funk", "Dinge"); } +// @Override public long expectedLength() { return 631; } +// @Override public long expectedTimestamp() { return TIMESTAMP; } +// @Override public long expectedViewCountAtLeast() { return 1_600_000; } +// @Nullable @Override public String expectedUploadDate() { return "2019-06-12 00:00:00.000"; } +// @Nullable @Override public String expectedTextualUploadDate() { return "2019-06-12"; } +// @Override public long expectedLikeCountAtLeast() { return 70000; } +// @Override public long expectedDislikeCountAtLeast() { return 500; } +// @Override public List expectedMetaInfo() throws MalformedURLException { +// return Collections.singletonList(new MetaInfo( +// EMPTY_STRING, +// new Description("Funk is a German public broadcast service.", Description.PLAIN_TEXT), +// Collections.singletonList(new URL("https://de.wikipedia.org/wiki/Funk_(Medienangebot)?wprov=yicw1")), +// Collections.singletonList("Wikipedia (German)") +// )); +// } +// @Override public boolean expectedUploaderVerified() { return true; } +// // @formatter:on +// @Override +// @Ignore("TODO fix") +// @Test +// public void testUploaderName() throws Exception { +// super.testUploaderName(); +// } +// +// @Override +// @Ignore("TODO fix") +// @Test +// public void testMetaInfo() throws Exception { +// super.testMetaInfo(); +// } +// +// } +// +//} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java index f498a573cf..b5067440f9 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java @@ -8,6 +8,7 @@ import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest; import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;