Skip to content

Extractor for youtube mix (auto-generated playlist) #280

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Stypox merged 14 commits into from
Dec 14, 2020
Merged

Extractor for youtube mix (auto-generated playlist) #280

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
0efb854
[Youtube] Implement mix extractor for auto-generated playlists.
XiangRongLin Feb 2, 2020
d74265c
[Youtube] Extract getThumbnailUrl into method and change getUploaderName
XiangRongLin Feb 2, 2020
327a573
[Youtube] Add some comments to mix
XiangRongLin Feb 2, 2020
a376792
[Youtube] Handle case where url is in "youtube.com/playlist?list=list…
XiangRongLin Feb 4, 2020
0ff054a
[Youtube] Extract initial playlist info
XiangRongLin Mar 6, 2020
ec6b99c
[Youtube] Adjust mix extractor to new user agent
XiangRongLin Mar 7, 2020
68a3948
[Youtube] Fix get banner url
XiangRongLin Mar 7, 2020
df38b19
[Youtube] Add tests and take thumbnail image always from first video …
XiangRongLin Mar 7, 2020
822cf30
[Youtube] Add _ITEMS constants and improve code style
XiangRongLin Mar 17, 2020
3ff8619
[Youtube] apply wb9688 suggestion (mix)
XiangRongLin Mar 21, 2020
4219354
[Youtube] Add subchannel functions to mix and fix imports
Stypox May 21, 2020
22d2f7e
[Youtube] Add cookies to youtube mix request
XiangRongLin Apr 16, 2020
a338e4e
[Youtube] Apply review suggestions and avoid channel mix edge case
XiangRongLin Sep 26, 2020
f90f6fc
[YouTube] Don't escape & in getUrlFromNavigationEndpoint for playlists
XiangRongLin Dec 12, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion ...xtractor/services/media_ccc/extractors/infoItems/MediaCCCConferenceInfoItemExtractor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
package org.schabi.newpipe.extractor.services.media_ccc.extractors.infoItems;

import com.grack.nanojson.JsonObject;

import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;

Expand All @@ -23,7 +25,7 @@ public long getSubscriberCount() {

@Override
public long getStreamCount() {
return -1;
return ListExtractor.ITEM_COUNT_UNKNOWN;
}

@Override
Expand Down
125 changes: 116 additions & 9 deletions ...tor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,10 @@
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import com.grack.nanojson.JsonWriter;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
Expand All @@ -21,6 +23,7 @@
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLDecoder;
import java.nio.charset.StandardCharsets;
import java.time.LocalDate;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
Expand All @@ -35,6 +38,7 @@
import static org.schabi.newpipe.extractor.utils.Utils.HTTP;
import static org.schabi.newpipe.extractor.utils.Utils.HTTPS;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import static org.schabi.newpipe.extractor.utils.Utils.join;

/*
* Created by Christian Schabesberger on 02.03.16.
Expand All @@ -61,6 +65,12 @@ public class YoutubeParsingHelper {
private YoutubeParsingHelper() {
}

/**
* The official youtube app supports intents in this format, where after the ':' is the videoId.
* Accordingly there are other apps sharing streams in this format.
*/
public final static String BASE_YOUTUBE_INTENT_URL = "vnd.youtube";

private static final String HARDCODED_CLIENT_VERSION = "2.20200214.04.00";
private static String clientVersion;

Expand Down Expand Up @@ -192,6 +202,57 @@ public static OffsetDateTime parseDateFrom(String textualUploadDate) throws Pars
}
}

/**
* Checks if the given playlist id is a YouTube Mix (auto-generated playlist)
* Ids from a YouTube Mix start with "RD"
* @param playlistId
* @return Whether given id belongs to a YouTube Mix
*/
public static boolean isYoutubeMixId(final String playlistId) {
return playlistId.startsWith("RD") && !isYoutubeMusicMixId(playlistId);
}

/**
* Checks if the given playlist id is a YouTube Music Mix (auto-generated playlist)
* Ids from a YouTube Music Mix start with "RDAMVM"
* @param playlistId
* @return Whether given id belongs to a YouTube Music Mix
*/
public static boolean isYoutubeMusicMixId(final String playlistId) {
return playlistId.startsWith("RDAMVM");
}
/**
* Checks if the given playlist id is a YouTube Channel Mix (auto-generated playlist)
* Ids from a YouTube channel Mix start with "RDCM"
* @return Whether given id belongs to a YouTube Channel Mix
*/
public static boolean isYoutubeChannelMixId(final String playlistId) {
return playlistId.startsWith("RDCM");
}

/**
* Extracts the video id from the playlist id for Mixes.
* @throws ParsingException If the playlistId is a Channel Mix or not a mix.
*/
public static String extractVideoIdFromMixId(final String playlistId) throws ParsingException {
if (playlistId.startsWith("RDMM")) { //My Mix
return playlistId.substring(4);

} else if (playlistId.startsWith("RDAMVM")) { //Music mix
return playlistId.substring(6);

} else if (playlistId.startsWith("RMCM")) { //Channel mix
//Channel mix are build with RMCM{channelId}, so videoId can't be determined
throw new ParsingException("Video id could not be determined from mix id: " + playlistId);

} else if (playlistId.startsWith("RD")) { // Normal mix
return playlistId.substring(2);

} else { //not a mix
throw new ParsingException("Video id could not be determined from mix id: " + playlistId);
}
}

public static JsonObject getInitialData(String html) throws ParsingException {
try {
try {
Expand Down Expand Up @@ -416,10 +477,14 @@ public static String getUrlFromNavigationEndpoint(JsonObject navigationEndpoint)
} else if (navigationEndpoint.has("watchEndpoint")) {
StringBuilder url = new StringBuilder();
url.append("https://www.youtube.com/watch?v=").append(navigationEndpoint.getObject("watchEndpoint").getString("videoId"));
if (navigationEndpoint.getObject("watchEndpoint").has("playlistId"))
url.append("&list=").append(navigationEndpoint.getObject("watchEndpoint").getString("playlistId"));
if (navigationEndpoint.getObject("watchEndpoint").has("startTimeSeconds"))
url.append("&t=").append(navigationEndpoint.getObject("watchEndpoint").getInt("startTimeSeconds"));
if (navigationEndpoint.getObject("watchEndpoint").has("playlistId")) {
url.append("&list=").append(navigationEndpoint.getObject("watchEndpoint")
.getString("playlistId"));
}
if (navigationEndpoint.getObject("watchEndpoint").has("startTimeSeconds")) {
url.append("&t=").append(navigationEndpoint.getObject("watchEndpoint")
.getInt("startTimeSeconds"));
}
return url.toString();
} else if (navigationEndpoint.has("watchPlaylistEndpoint")) {
return "https://www.youtube.com/playlist?list=" +
Expand Down Expand Up @@ -485,8 +550,8 @@ public static String fixThumbnailUrl(String thumbnailUrl) {
public static String getValidJsonResponseBody(final Response response)
throws ParsingException, MalformedURLException {
if (response.responseCode() == 404) {
throw new ContentNotAvailableException("Not found" +
" (\"" + response.responseCode() + " " + response.responseMessage() + "\")");
throw new ContentNotAvailableException("Not found"
+ " (\"" + response.responseCode() + " " + response.responseMessage() + "\")");
}

final String responseBody = response.responseBody();
Expand All @@ -506,22 +571,64 @@ public static String getValidJsonResponseBody(final Response response)
final String responseContentType = response.getHeader("Content-Type");
if (responseContentType != null
&& responseContentType.toLowerCase().contains("text/html")) {
throw new ParsingException("Got HTML document, expected JSON response" +
" (latest url was: \"" + response.latestUrl() + "\")");
throw new ParsingException("Got HTML document, expected JSON response"
+ " (latest url was: \"" + response.latestUrl() + "\")");
}

return responseBody;
}

public static Response getResponse(final String url, final Localization localization)
throws IOException, ExtractionException {
final Map<String, List<String>> headers = new HashMap<>();
headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion()));

final Response response = getDownloader().get(url, headers, localization);
getValidJsonResponseBody(response);

return response;
}

public static String extractCookieValue(final String cookieName, final Response response) {
final List<String> cookies = response.responseHeaders().get("Set-Cookie");
int startIndex;
String result = "";
for (final String cookie : cookies) {
startIndex = cookie.indexOf(cookieName);
if (startIndex != -1) {
result = cookie.substring(startIndex + cookieName.length() + "=".length(),
cookie.indexOf(";", startIndex));
}
}
return result;
}

public static JsonArray getJsonResponse(final String url, final Localization localization)
throws IOException, ExtractionException {
Map<String, List<String>> headers = new HashMap<>();
headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion()));
final Response response = getDownloader().get(url, headers, localization);

final String responseBody = getValidJsonResponseBody(response);
return toJsonArray(getValidJsonResponseBody(response));
}

public static JsonArray getJsonResponse(final Page page, final Localization localization)
throws IOException, ExtractionException {
final Map<String, List<String>> headers = new HashMap<>();
if (!isNullOrEmpty(page.getCookies())) {
headers.put("Cookie", Collections.singletonList(join(";", "=", page.getCookies())));
}
headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion()));

final Response response = getDownloader().get(page.getUrl(), headers, localization);

return toJsonArray(getValidJsonResponseBody(response));
}

public static JsonArray toJsonArray(final String responseBody) throws ParsingException {
try {
return JsonParser.array().from(responseBody);
} catch (JsonParserException e) {
Expand Down
9 changes: 7 additions & 2 deletions extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeCommentsExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeFeedExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMixPlaylistExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMusicSearchExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubePlaylistExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor;
Expand Down Expand Up @@ -109,8 +110,12 @@ public ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler) {
}

@Override
public PlaylistExtractor getPlaylistExtractor(ListLinkHandler linkHandler) {
return new YoutubePlaylistExtractor(this, linkHandler);
public PlaylistExtractor getPlaylistExtractor(final ListLinkHandler linkHandler) {
if (YoutubeParsingHelper.isYoutubeMixId(linkHandler.getId())) {
return new YoutubeMixPlaylistExtractor(this, linkHandler);
} else {
return new YoutubePlaylistExtractor(this, linkHandler);
}
}

@Override
Expand Down
3 changes: 2 additions & 1 deletion ...schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import com.grack.nanojson.JsonObject;

import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
Expand Down Expand Up @@ -86,7 +87,7 @@ public long getStreamCount() throws ParsingException {
try {
if (!channelInfoItem.has("videoCountText")) {
// Video count is not available, channel probably has no public uploads.
return -1;
return ListExtractor.ITEM_COUNT_UNKNOWN;
}

return Long.parseLong(Utils.removeNonDigitCharacters(getTextFromObject(
Expand Down
Loading