Skip to content

Support YouTube's learning playlists #332

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
TobiGr merged 2 commits into from
May 29, 2020
Merged

Support YouTube's learning playlists #332

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4d683e7
Support YouTube's learning playlists
wb9688 May 13, 2020
ab77961
Extract trailer thumbnail in YT learning playlists
wb9688 May 13, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
121 changes: 105 additions & 16 deletions ...va/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,16 +8,21 @@
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.utils.Utils;

import java.io.IOException;

import javax.annotation.Nonnull;
import javax.annotation.Nullable;

import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse;
Expand All @@ -27,6 +32,7 @@

@SuppressWarnings("WeakerAccess")
public class YoutubePlaylistExtractor extends PlaylistExtractor {
private JsonArray initialAjaxJson;
private JsonObject initialData;
private JsonObject playlistInfo;

Expand All @@ -38,9 +44,9 @@ public YoutubePlaylistExtractor(StreamingService service, ListLinkHandler linkHa
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
final String url = getUrl() + "&pbj=1";

final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization());
initialAjaxJson = getJsonResponse(url, getExtractorLocalization());

initialData = ajaxJson.getObject(1).getObject("response");
initialData = initialAjaxJson.getObject(1).getObject("response");
YoutubeParsingHelper.defaultAlertsCheck(initialData);

playlistInfo = getPlaylistInfo();
Expand Down Expand Up @@ -152,34 +158,47 @@ public long getStreamCount() throws ParsingException {

@Nonnull
@Override
public String getSubChannelName() throws ParsingException {
public String getSubChannelName() {
return "";
}

@Nonnull
@Override
public String getSubChannelUrl() throws ParsingException {
public String getSubChannelUrl() {
return "";
}

@Nonnull
@Override
public String getSubChannelAvatarUrl() throws ParsingException {
public String getSubChannelAvatarUrl() {
return "";
}

@Nonnull
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() {
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());

JsonArray videos = initialData.getObject("contents").getObject("twoColumnBrowseResultsRenderer")
final JsonArray contents = initialData.getObject("contents").getObject("twoColumnBrowseResultsRenderer")
.getArray("tabs").getObject(0).getObject("tabRenderer").getObject("content")
.getObject("sectionListRenderer").getArray("contents").getObject(0)
.getObject("itemSectionRenderer").getArray("contents").getObject(0)
.getObject("playlistVideoListRenderer").getArray("contents");
.getObject("itemSectionRenderer").getArray("contents");

if (contents.getObject(0).has("playlistSegmentRenderer")) {
for (final Object segment : contents) {
if (((JsonObject) segment).getObject("playlistSegmentRenderer").has("trailer")) {
collectTrailerFrom(collector, ((JsonObject) segment));
} else if (((JsonObject) segment).getObject("playlistSegmentRenderer").has("videoList")) {
collectStreamsFrom(collector, ((JsonObject) segment).getObject("playlistSegmentRenderer")
.getObject("videoList").getObject("playlistVideoListRenderer").getArray("contents"));
}
}
} else if (contents.getObject(0).has("playlistVideoListRenderer")) {
final JsonArray videos = contents.getObject(0)
.getObject("playlistVideoListRenderer").getArray("contents");
collectStreamsFrom(collector, videos);
}

collectStreamsFrom(collector, videos);
return new InfoItemsPage<>(collector, getNextPageUrl());
}

Expand All @@ -189,18 +208,18 @@ public InfoItemsPage<StreamInfoItem> getPage(final String pageUrl) throws IOExce
throw new ExtractionException(new IllegalArgumentException("Page url is empty or null"));
}

StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization());

JsonObject sectionListContinuation = ajaxJson.getObject(1).getObject("response")
final JsonObject sectionListContinuation = ajaxJson.getObject(1).getObject("response")
.getObject("continuationContents").getObject("playlistVideoListContinuation");

collectStreamsFrom(collector, sectionListContinuation.getArray("contents"));

return new InfoItemsPage<>(collector, getNextPageUrlFrom(sectionListContinuation.getArray("continuations")));
}

private String getNextPageUrlFrom(JsonArray continuations) {
private String getNextPageUrlFrom(final JsonArray continuations) {
if (isNullOrEmpty(continuations)) {
return "";
}
Expand All @@ -212,9 +231,7 @@ private String getNextPageUrlFrom(JsonArray continuations) {
+ "&itct=" + clickTrackingParams;
}

private void collectStreamsFrom(StreamInfoItemsCollector collector, JsonArray videos) {
collector.reset();

private void collectStreamsFrom(final StreamInfoItemsCollector collector, final JsonArray videos) {
final TimeAgoParser timeAgoParser = getTimeAgoParser();

for (Object video : videos) {
Expand All @@ -228,4 +245,76 @@ public long getViewCount() {
}
}
}

private void collectTrailerFrom(final StreamInfoItemsCollector collector,
final JsonObject segment) {
collector.commit(new StreamInfoItemExtractor() {
@Override
public String getName() throws ParsingException {
return getTextFromObject(segment.getObject("playlistSegmentRenderer")
.getObject("title"));
}

@Override
public String getUrl() throws ParsingException {
return YoutubeStreamLinkHandlerFactory.getInstance()
.fromId(segment.getObject("playlistSegmentRenderer").getObject("trailer")
.getObject("playlistVideoPlayerRenderer").getString("videoId"))
.getUrl();
}

@Override
public String getThumbnailUrl() {
final JsonArray thumbnails = initialAjaxJson.getObject(1).getObject("playerResponse")
.getObject("videoDetails").getObject("thumbnail").getArray("thumbnails");
// the last thumbnail is the one with the highest resolution
final String url = thumbnails.getObject(thumbnails.size() - 1).getString("url");
return fixThumbnailUrl(url);
}

@Override
public StreamType getStreamType() {
return StreamType.VIDEO_STREAM;
}

@Override
public boolean isAd() {
return false;
}

@Override
public long getDuration() throws ParsingException {
return YoutubeParsingHelper.parseDurationString(
getTextFromObject(segment.getObject("playlistSegmentRenderer")
.getObject("segmentAnnotation")).split("•")[0]);
}

@Override
public long getViewCount() {
return -1;
}

@Override
public String getUploaderName() throws ParsingException {
return YoutubePlaylistExtractor.this.getUploaderName();
}

@Override
public String getUploaderUrl() throws ParsingException {
return YoutubePlaylistExtractor.this.getUploaderUrl();
}

@Nullable
@Override
public String getTextualUploadDate() {
return null;
}

@Nullable
@Override
public DateWrapper getUploadDate() {
return null;
}
});
}
}
98 changes: 98 additions & 0 deletions ...test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -254,4 +254,102 @@ public void testStreamCount() throws Exception {
assertTrue("Error in the streams count", extractor.getStreamCount() > 100);
}
}

public static class LearningPlaylist implements BasePlaylistExtractorTest {
private static YoutubePlaylistExtractor extractor;

@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (YoutubePlaylistExtractor) YouTube
.getPlaylistExtractor("https://www.youtube.com/playlist?list=PL8dPuuaLjXtOAKed_MxxWBNaPno5h3Zs8");
extractor.fetchPage();
}

/*//////////////////////////////////////////////////////////////////////////
// Extractor
//////////////////////////////////////////////////////////////////////////*/

@Test
public void testServiceId() {
assertEquals(YouTube.getServiceId(), extractor.getServiceId());
}

@Test
public void testName() throws Exception {
String name = extractor.getName();
assertTrue(name, name.startsWith("Anatomy & Physiology"));
}

@Test
public void testId() throws Exception {
assertEquals("PL8dPuuaLjXtOAKed_MxxWBNaPno5h3Zs8", extractor.getId());
}

@Test
public void testUrl() throws ParsingException {
assertEquals("https://www.youtube.com/playlist?list=PL8dPuuaLjXtOAKed_MxxWBNaPno5h3Zs8", extractor.getUrl());
}

@Test
public void testOriginalUrl() throws ParsingException {
assertEquals("https://www.youtube.com/playlist?list=PL8dPuuaLjXtOAKed_MxxWBNaPno5h3Zs8", extractor.getOriginalUrl());
}

/*//////////////////////////////////////////////////////////////////////////
// ListExtractor
//////////////////////////////////////////////////////////////////////////*/

@Test
public void testRelatedItems() throws Exception {
defaultTestRelatedItems(extractor);
}

@Ignore
@Test
public void testMoreRelatedItems() throws Exception {
defaultTestMoreItems(extractor);
}

/*//////////////////////////////////////////////////////////////////////////
// PlaylistExtractor
//////////////////////////////////////////////////////////////////////////*/

@Test
public void testThumbnailUrl() throws Exception {
final String thumbnailUrl = extractor.getThumbnailUrl();
assertIsSecureUrl(thumbnailUrl);
assertTrue(thumbnailUrl, thumbnailUrl.contains("yt"));
}

@Ignore
@Test
public void testBannerUrl() throws Exception {
final String bannerUrl = extractor.getBannerUrl();
assertIsSecureUrl(bannerUrl);
assertTrue(bannerUrl, bannerUrl.contains("yt"));
}

@Test
public void testUploaderUrl() throws Exception {
assertEquals("https://www.youtube.com/channel/UCX6b17PVsYBQ0ip5gyeme-Q", extractor.getUploaderUrl());
}

@Test
public void testUploaderName() throws Exception {
final String uploaderName = extractor.getUploaderName();
assertTrue(uploaderName, uploaderName.contains("CrashCourse"));
}

@Test
public void testUploaderAvatarUrl() throws Exception {
final String uploaderAvatarUrl = extractor.getUploaderAvatarUrl();
assertTrue(uploaderAvatarUrl, uploaderAvatarUrl.contains("yt"));
}

@Test
public void testStreamCount() throws Exception {
assertTrue("Error in the streams count", extractor.getStreamCount() > 40);
}
}
}