diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java index 6137f02931..3c73f66fb0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java @@ -28,6 +28,7 @@ import org.schabi.newpipe.extractor.services.youtube.extractors.*; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeCommentsLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory; @@ -101,7 +102,11 @@ public ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler) { @Override public PlaylistExtractor getPlaylistExtractor(ListLinkHandler linkHandler) { - return new YoutubePlaylistExtractor(this, linkHandler); + if (YoutubeParsingHelper.isYoutubeMixId(linkHandler.getId())) { + return new YoutubeMixPlaylistExtractor(this, linkHandler); + } else { + return new YoutubePlaylistExtractor(this, linkHandler); + } } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java new file mode 100644 index 0000000000..9da2e98fdd --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java @@ -0,0 +1,205 @@ +package org.schabi.newpipe.extractor.services.youtube.extractors; + +import java.io.IOException; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.downloader.Response; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.localization.TimeAgoParser; +import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; + +/** + * A YoutubePlaylistExtractor for a mix (auto-generated playlist). + * It handles urls in the format of "youtube.com/watch?v=videoId&list=playlistId" + */ +public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { + + private Document doc; + + public YoutubeMixPlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) { + super(service, linkHandler); + } + + @Override + public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { + final String url = getUrl(); + final Response response = downloader.get(url, getExtractorLocalization()); + doc = YoutubeParsingHelper.parseAndCheckPage(url, response); + } + + @Nonnull + @Override + public String getName() throws ParsingException { + try { + return doc.select("div[class=\"playlist-info\"] h3[class=\"playlist-title\"]").first().text(); + } catch (Exception e) { + throw new ParsingException("Could not get playlist name", e); + } + } + + @Override + public String getThumbnailUrl() throws ParsingException { + try { + Element li = doc.select("ol[class*=\"playlist-videos-list\"] li").first(); + String videoId = li.attr("data-video-id"); + if (videoId != null && !videoId.isEmpty()) { + //higher quality + return getThumbnailUrlFromId(videoId); + } else { + //lower quality + return doc.select("ol[class*=\"playlist-videos-list\"] li").first() + .attr("data-thumbnail-url"); + } + } catch (Exception e) { + throw new ParsingException("Could not get playlist thumbnail", e); + } + } + + @Override + public String getBannerUrl() { + return ""; + } + + @Override + public String getUploaderUrl() { + //Youtube mix are auto-generated + return ""; + } + + @Override + public String getUploaderName() { + //Youtube mix are auto-generated + return ""; + } + + @Override + public String getUploaderAvatarUrl() { + //Youtube mix are auto-generated + return ""; + } + + @Override + public long getStreamCount() { + // Auto-generated playlist always start with 25 videos and are endless + // But the html doesn't have a continuation url + return doc.select("ol[class*=\"playlist-videos-list\"] li").size(); + } + + @Nonnull + @Override + public InfoItemsPage getInitialPage() { + StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + Element ol = doc.select("ol[class*=\"playlist-videos-list\"]").first(); + collectStreamsFrom(collector, ol); + return new InfoItemsPage<>(collector, getNextPageUrl()); + } + + @Override + public String getNextPageUrl() { + return ""; + } + + @Override + public InfoItemsPage getPage(final String pageUrl) { + //Continuations are not implemented + return null; + } + + private void collectStreamsFrom( + @Nonnull StreamInfoItemsCollector collector, + @Nullable Element element) { + collector.reset(); + + if (element == null) { + return; + } + + final LinkHandlerFactory streamLinkHandlerFactory = getService().getStreamLHFactory(); + final TimeAgoParser timeAgoParser = getTimeAgoParser(); + + for (final Element li : element.children()) { + + collector.commit(new YoutubeStreamInfoItemExtractor(li, timeAgoParser) { + + @Override + public boolean isAd() { + return false; + } + + @Override + public String getUrl() throws ParsingException { + try { + return streamLinkHandlerFactory.fromId(li.attr("data-video-id")).getUrl(); + } catch (Exception e) { + throw new ParsingException("Could not get web page url for the video", e); + } + } + + @Override + public String getName() throws ParsingException { + try { + return li.attr("data-video-title"); + } catch (Exception e) { + throw new ParsingException("Could not get name", e); + } + } + + @Override + public long getDuration() { + //Not present in doc + return 0; + } + + @Override + public String getUploaderName() throws ParsingException { + String uploaderName = li.attr("data-video-username"); + if (uploaderName == null || uploaderName.isEmpty()) { + throw new ParsingException("Could not get uploader name"); + } else { + return uploaderName; + } + } + + @Override + public String getUploaderUrl() { + //Not present in doc + return ""; + } + + @Override + public String getTextualUploadDate() { + //Not present in doc + return ""; + } + + @Override + public long getViewCount() { + return -1; + } + + @Override + public String getThumbnailUrl() throws ParsingException { + try { + return getThumbnailUrlFromId(streamLinkHandlerFactory.fromUrl(getUrl()).getId()); + } catch (Exception e) { + throw new ParsingException("Could not get thumbnail url", e); + } + } + }); + } + } + + private String getThumbnailUrlFromId(String videoId) { + return "https://i.ytimg.com/vi/" + videoId + "/hqdefault.jpg"; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java index 65ec7e3f68..f9ba1d6544 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java @@ -143,4 +143,14 @@ public static Calendar parseDateFrom(String textualUploadDate) throws ParsingExc uploadDate.setTime(date); return uploadDate; } + + /** + * Checks if the given playlist id is a mix (auto-generated playlist) + * Ids from a mix start with "RD" + * @param playlistId + * @return Whether given id belongs to a mix + */ + public static boolean isYoutubeMixId(String playlistId) { + return playlistId.startsWith("RD"); + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java index 62a0e73751..34d4d8d270 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java @@ -1,6 +1,9 @@ package org.schabi.newpipe.extractor.services.youtube.linkHandler; +import java.net.MalformedURLException; import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.LinkHandler; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; import org.schabi.newpipe.extractor.utils.Utils; @@ -60,4 +63,30 @@ public boolean onAcceptUrl(final String url) { } return true; } + + /** + * If it is a mix (auto-generated playlist) url, return a Linkhandler where the url is like + * youtube.com/watch?v=videoId&list=playlistId + *

Otherwise use super

+ */ + @Override + public ListLinkHandler fromUrl(String url) throws ParsingException { + try { + URL urlObj = Utils.stringToURL(url); + String listID = Utils.getQueryValue(urlObj, "list"); + if (listID != null && YoutubeParsingHelper.isYoutubeMixId(listID)) { + String videoID = Utils.getQueryValue(urlObj, "v"); + if (videoID == null) { + videoID = listID.substring(2); + } + String newUrl = "https://www.youtube.com/watch?v=" + videoID + "&list=" + listID; + return new ListLinkHandler(new LinkHandler(url, newUrl, listID), getContentFilter(url), + getSortFilter(url)); + } + } catch (MalformedURLException exception) { + throw new ParsingException("Error could not parse url :" + exception.getMessage(), + exception); + } + return super.fromUrl(url); + } }