Skip to content

Polish #442

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
TobiGr merged 2 commits into from
Oct 26, 2020
Merged

Polish #442

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
01f49e8
polish strings
B0pol Oct 26, 2020
0a12300
polish tests
B0pol Oct 26, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
81 changes: 42 additions & 39 deletions ...java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;

import org.mozilla.javascript.Context;
import org.mozilla.javascript.Function;
import org.mozilla.javascript.ScriptableObject;
Expand Down Expand Up @@ -36,6 +35,8 @@
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Utils;

import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.text.SimpleDateFormat;
Expand All @@ -49,9 +50,6 @@
import java.util.Locale;
import java.util.Map;

import javax.annotation.Nonnull;
import javax.annotation.Nullable;

import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
Expand Down Expand Up @@ -84,8 +82,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
// Exceptions
//////////////////////////////////////////////////////////////////////////*/

public class DecryptException extends ParsingException {
DecryptException(String message, Throwable cause) {
public class DeobfuscateException extends ParsingException {
DeobfuscateException(String message, Throwable cause) {
super(message, cause);
}
}
Expand Down Expand Up @@ -156,20 +154,23 @@ public String getTextualUploadDate() throws ParsingException {
TimeAgoParser timeAgoParser = TimeAgoPatternsManager.getTimeAgoParserFor(Localization.fromLocalizationCode("en"));
Calendar parsedTime = timeAgoParser.parse(time).date();
return new SimpleDateFormat("yyyy-MM-dd").format(parsedTime.getTime());
} catch (Exception ignored) {}
} catch (Exception ignored) {
}

try { // Premiered Feb 21, 2020
Date d = new SimpleDateFormat("MMM dd, YYYY", Locale.ENGLISH).parse(time);
return new SimpleDateFormat("yyyy-MM-dd").format(d.getTime());
} catch (Exception ignored) {}
} catch (Exception ignored) {
}
}

try {
// TODO: this parses English formatted dates only, we need a better approach to parse the textual date
Date d = new SimpleDateFormat("dd MMM yyyy", Locale.ENGLISH).parse(
getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText")));
return new SimpleDateFormat("yyyy-MM-dd").format(d);
} catch (Exception ignored) {}
} catch (Exception ignored) {
}
throw new ParsingException("Could not get upload date");
}

Expand Down Expand Up @@ -368,7 +369,8 @@ public String getUploaderName() throws ParsingException {
try {
uploaderName = getTextFromObject(getVideoSecondaryInfoRenderer().getObject("owner")
.getObject("videoOwnerRenderer").getObject("title"));
} catch (ParsingException ignored) { }
} catch (ParsingException ignored) {
}

if (isNullOrEmpty(uploaderName)) {
uploaderName = playerResponse.getObject("videoDetails").getString("author");
Expand Down Expand Up @@ -436,11 +438,11 @@ public String getDashMpdUrl() throws ParsingException {
}

if (!dashManifestUrl.contains("/signature/")) {
String encryptedSig = Parser.matchGroup1("/s/([a-fA-F0-9\\.]+)", dashManifestUrl);
String decryptedSig;
String obfuscatedSig = Parser.matchGroup1("/s/([a-fA-F0-9\\.]+)", dashManifestUrl);
String deobfuscatedSig;

decryptedSig = decryptSignature(encryptedSig, decryptionCode);
dashManifestUrl = dashManifestUrl.replace("/s/" + encryptedSig, "/signature/" + decryptedSig);
deobfuscatedSig = deobfuscateSignature(obfuscatedSig, deobfuscationCode);
dashManifestUrl = dashManifestUrl.replace("/s/" + obfuscatedSig, "/signature/" + deobfuscatedSig);
}

return dashManifestUrl;
Expand Down Expand Up @@ -630,7 +632,7 @@ public String getErrorMessage() {
private static final String FORMATS = "formats";
private static final String ADAPTIVE_FORMATS = "adaptiveFormats";
private static final String HTTPS = "https:";
private static final String DECRYPTION_FUNC_NAME = "decrypt";
private static final String DEOBFUSCATION_FUNC_NAME = "decrypt";

private final static String[] REGEXES = {
"(?:\\b|[^a-zA-Z0-9$])([a-zA-Z0-9$]{2})\\s*=\\s*function\\(\\s*a\\s*\\)\\s*\\{\\s*a\\s*=\\s*a\\.split\\(\\s*\"\"\\s*\\)",
Expand All @@ -640,7 +642,7 @@ public String getErrorMessage() {
"\\bc\\s*&&\\s*d\\.set\\([^,]+\\s*,\\s*(:encodeURIComponent\\s*\\()([a-zA-Z0-9$]+)\\("
};

private volatile String decryptionCode = "";
private volatile String deobfuscationCode = "";

@Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
Expand Down Expand Up @@ -695,8 +697,8 @@ public void onFetchPage(@Nonnull Downloader downloader) throws IOException, Extr
throw new ContentNotAvailableException("Got error: \"" + reason + "\"");
}

if (decryptionCode.isEmpty()) {
decryptionCode = loadDecryptionCode(playerUrl);
if (deobfuscationCode.isEmpty()) {
deobfuscationCode = loadDeobfuscationCode(playerUrl);
}

if (subtitlesInfos.isEmpty()) {
Expand All @@ -716,7 +718,7 @@ private JsonObject getPlayerArgs(final JsonObject playerConfig) throws ParsingEx
private String getPlayerUrl(final JsonObject playerConfig) throws ParsingException {
// The Youtube service needs to be initialized by downloading the
// js-Youtube-player. This is done in order to get the algorithm
// for decrypting cryptic signatures inside certain stream URLs.
// for deobfuscating cryptic signatures inside certain stream URLs.
final String playerUrl = playerConfig.getObject("assets").getString("js");

if (playerUrl == null) {
Expand Down Expand Up @@ -768,11 +770,11 @@ private EmbeddedInfo getEmbeddedInfo() throws ParsingException, ReCaptchaExcepti

} catch (IOException e) {
throw new ParsingException(
"Could load decryption code form restricted video for the Youtube service.", e);
"Could load deobfuscation code form restricted video for the Youtube service.", e);
Copy link
Member

@Stypox Stypox Oct 26, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Mmmh, this should probably be "Could not load deobfuscation code for YouTube restricted video"

Copy link
Member Author

@B0pol B0pol Oct 26, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sounds good. Feel free to open a PR

Copy link
Member

@Stypox Stypox Oct 26, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

#444

}
}

private String loadDecryptionCode(String playerUrl) throws DecryptException {
private String loadDeobfuscationCode(String playerUrl) throws DeobfuscateException {
try {
Downloader downloader = NewPipe.getDownloader();
if (!playerUrl.contains("https://youtube.com")) {
Expand All @@ -782,49 +784,49 @@ private String loadDecryptionCode(String playerUrl) throws DecryptException {
}

final String playerCode = downloader.get(playerUrl, getExtractorLocalization()).responseBody();
final String decryptionFunctionName = getDecryptionFuncName(playerCode);
final String deobfuscationFunctionName = getDeobfuscationFuncName(playerCode);

final String functionPattern = "("
+ decryptionFunctionName.replace("$", "\\$")
+ deobfuscationFunctionName.replace("$", "\\$")
+ "=function\\([a-zA-Z0-9_]+\\)\\{.+?\\})";
final String decryptionFunction = "var " + Parser.matchGroup1(functionPattern, playerCode) + ";";
final String deobfuscateFunction = "var " + Parser.matchGroup1(functionPattern, playerCode) + ";";

final String helperObjectName =
Parser.matchGroup1(";([A-Za-z0-9_\\$]{2})\\...\\(", decryptionFunction);
Parser.matchGroup1(";([A-Za-z0-9_\\$]{2})\\...\\(", deobfuscateFunction);
final String helperPattern =
"(var " + helperObjectName.replace("$", "\\$") + "=\\{.+?\\}\\};)";
final String helperObject =
Parser.matchGroup1(helperPattern, playerCode.replace("\n", ""));

final String callerFunction =
"function " + DECRYPTION_FUNC_NAME + "(a){return " + decryptionFunctionName + "(a);}";
"function " + DEOBFUSCATION_FUNC_NAME + "(a){return " + deobfuscationFunctionName + "(a);}";

return helperObject + decryptionFunction + callerFunction;
return helperObject + deobfuscateFunction + callerFunction;
} catch (IOException ioe) {
throw new DecryptException("Could not load decrypt function", ioe);
throw new DeobfuscateException("Could not load deobfuscate function", ioe);
} catch (Exception e) {
throw new DecryptException("Could not parse decrypt function ", e);
throw new DeobfuscateException("Could not parse deobfuscate function ", e);
}
}

private String decryptSignature(String encryptedSig, String decryptionCode) throws DecryptException {
private String deobfuscateSignature(String obfuscatedSig, String deobfuscationCode) throws DeobfuscateException {
final Context context = Context.enter();
context.setOptimizationLevel(-1);
final Object result;
try {
final ScriptableObject scope = context.initSafeStandardObjects();
context.evaluateString(scope, decryptionCode, "decryptionCode", 1, null);
final Function decryptionFunc = (Function) scope.get("decrypt", scope);
result = decryptionFunc.call(context, scope, scope, new Object[]{encryptedSig});
context.evaluateString(scope, deobfuscationCode, "decryptionCode", 1, null);
final Function deobfuscateFunc = (Function) scope.get("decrypt", scope);
result = deobfuscateFunc.call(context, scope, scope, new Object[]{obfuscatedSig});
} catch (Exception e) {
throw new DecryptException("Could not get decrypt signature", e);
throw new DeobfuscateException("Could not get deobfuscate signature", e);
} finally {
Context.exit();
}
return result == null ? "" : result.toString();
}

private String getDecryptionFuncName(final String playerCode) throws DecryptException {
private String getDeobfuscationFuncName(final String playerCode) throws DeobfuscateException {
Parser.RegexException exception = null;
for (final String regex : REGEXES) {
try {
Expand All @@ -835,7 +837,7 @@ private String getDecryptionFuncName(final String playerCode) throws DecryptExce
}
}
}
throw new DecryptException("Could not find decrypt function with any of the given patterns.", exception);
throw new DeobfuscateException("Could not find deobfuscate function with any of the given patterns.", exception);
}

@Nonnull
Expand Down Expand Up @@ -989,18 +991,19 @@ private Map<String, ItagItem> getItags(String streamingDataKey, ItagItem.ItagTyp
if (formatData.has("url")) {
streamUrl = formatData.getString("url");
} else {
// this url has an encrypted signature
// this url has an obfuscated signature
final String cipherString = formatData.has("cipher")
? formatData.getString("cipher")
: formatData.getString("signatureCipher");
final Map<String, String> cipher = Parser.compatParseMap(cipherString);
streamUrl = cipher.get("url") + "&" + cipher.get("sp") + "="
+ decryptSignature(cipher.get("s"), decryptionCode);
+ deobfuscateSignature(cipher.get("s"), deobfuscationCode);
}

urlAndItags.put(streamUrl, itagItem);
}
} catch (UnsupportedEncodingException ignored) {}
} catch (UnsupportedEncodingException ignored) {
}
}
}

Expand Down
45 changes: 12 additions & 33 deletions .../java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,9 @@

public class SoundcloudStreamExtractorTest {

public static class LilUziVertDoWhatIWant extends DefaultStreamExtractorTest {
private static final String ID = "do-what-i-want-produced-by-maaly-raw-don-cannon";
private static final String UPLOADER = "https://soundcloud.com/liluzivert";
public static class CreativeCommonsPlaysWellWithOthers extends DefaultStreamExtractorTest {
private static final String ID = "plays-well-with-others-ep-2-what-do-an-army-of-ants-and-an-online-encyclopedia-have-in-common";
private static final String UPLOADER = "https://soundcloud.com/wearecc";
private static final int TIMESTAMP = 69;
private static final String URL = UPLOADER + "/" + ID + "#t=" + TIMESTAMP;
private static StreamExtractor extractor;
Expand All @@ -35,47 +35,26 @@ public static void setUp() throws Exception {

@Override public StreamExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return SoundCloud; }
@Override public String expectedName() { return "Do What I Want [Produced By Maaly Raw + Don Cannon]"; }
@Override public String expectedId() { return "276206960"; }
@Override public String expectedName() { return "Plays Well with Others, Ep 2: What Do an Army of Ants and an Online Encyclopedia Have in Common?"; }
@Override public String expectedId() { return "597253485"; }
@Override public String expectedUrlContains() { return UPLOADER + "/" + ID; }
@Override public String expectedOriginalUrlContains() { return URL; }

@Override public StreamType expectedStreamType() { return StreamType.AUDIO_STREAM; }
@Override public String expectedUploaderName() { return "Lil Uzi Vert"; }
@Override public String expectedUploaderName() { return "Creative Commons"; }
@Override public String expectedUploaderUrl() { return UPLOADER; }
@Override public List<String> expectedDescriptionContains() { return Arrays.asList("The Perfect LUV Tape®"); }
@Override public long expectedLength() { return 175; }
@Override public List<String> expectedDescriptionContains() { return Arrays.asList("Stigmergy is a mechanism of indirect coordination",
"All original content in Plays Well with Others is available under a Creative Commons BY license."); }
@Override public long expectedLength() { return 1400; }
@Override public long expectedTimestamp() { return TIMESTAMP; }
@Override public long expectedViewCountAtLeast() { return 75413600; }
@Nullable @Override public String expectedUploadDate() { return "2016-07-31 18:18:07.000"; }
@Nullable @Override public String expectedTextualUploadDate() { return "2016-07-31 18:18:07"; }
@Override public long expectedViewCountAtLeast() { return 27000; }
@Nullable @Override public String expectedUploadDate() { return "2019-03-28 13:36:18.000"; }
@Nullable @Override public String expectedTextualUploadDate() { return "2019-03-28 13:36:18"; }
@Override public long expectedLikeCountAtLeast() { return -1; }
@Override public long expectedDislikeCountAtLeast() { return -1; }
@Override public boolean expectedHasVideoStreams() { return false; }
@Override public boolean expectedHasSubtitles() { return false; }
@Override public boolean expectedHasFrames() { return false; }
}

public static class ContentNotSupported {
@BeforeClass
public static void setUp() {
NewPipe.init(DownloaderTestImpl.getInstance());
}

@Test(expected = ContentNotSupportedException.class)
public void hlsAudioStream() throws Exception {
final StreamExtractor extractor =
SoundCloud.getStreamExtractor("https://soundcloud.com/dualipa/cool");
extractor.fetchPage();
extractor.getAudioStreams();
}

@Test(expected = ContentNotSupportedException.class)
public void bothHlsAndOpusAudioStreams() throws Exception {
final StreamExtractor extractor =
SoundCloud.getStreamExtractor("https://soundcloud.com/lil-baby-4pf/no-sucker");
extractor.fetchPage();
extractor.getAudioStreams();
}
}
}
Loading