Release v0.17.0

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelInfoItemExtractor.java

@@ -24,7 +24,9 @@ public String getUrl() {
 
     @Override
     public String getThumbnailUrl() {
-        return itemObject.getString("avatar_url", "");
+        String avatarUrl = itemObject.getString("avatar_url", "");
+        String avatarUrlBetterResolution = avatarUrl.replace("large.jpg", "crop.jpg");
+        return avatarUrlBetterResolution;
     }
 
     @Override

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java

@@ -113,7 +113,10 @@ public static String resolveIdWithEmbedPlayer(String url) throws IOException, Re
 
         String response = NewPipe.getDownloader().download("https://w.soundcloud.com/player/?url="
                 + URLEncoder.encode(url, "UTF-8"));
-        return Parser.matchGroup1(",\"id\":(.*?),", response);
+        // handle playlists / sets different and get playlist id via uir field in JSON
+        if (url.contains("sets") && !url.endsWith("sets") && !url.endsWith("sets/"))
+            return Parser.matchGroup1("\"uri\":\\s*\"https:\\/\\/api\\.soundcloud\\.com\\/playlists\\/((\\d)*?)\"", response);
+        return Parser.matchGroup1(",\"id\":(([^}\\n])*?),", response);
     }
 
     /**

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java

@@ -71,13 +71,15 @@ public String getThumbnailUrl() {
                     final String thumbnailUrl = item.getThumbnailUrl();
                     if (thumbnailUrl == null || thumbnailUrl.isEmpty()) continue;
 
-                    return thumbnailUrl;
+                    String thumbnailUrlBetterResolution = thumbnailUrl.replace("large.jpg", "crop.jpg");
+                    return thumbnailUrlBetterResolution;
                 }
             } catch (Exception ignored) {
             }
         }
 
-        return artworkUrl;
+        String artworkUrlBetterResolution = artworkUrl.replace("large.jpg", "crop.jpg");
+        return artworkUrlBetterResolution;
     }
 
     @Override

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistInfoItemExtractor.java

@@ -32,7 +32,10 @@ public String getThumbnailUrl() throws ParsingException {
         // Over-engineering at its finest
         if (itemObject.isString(ARTWORK_URL_KEY)) {
             final String artworkUrl = itemObject.getString(ARTWORK_URL_KEY, "");
-            if (!artworkUrl.isEmpty()) return artworkUrl;
+            if (!artworkUrl.isEmpty()) {
+                String artworkUrlBetterResolution = artworkUrl.replace("large.jpg", "crop.jpg");
+                return artworkUrlBetterResolution;
+            }
         }
 
         try {
@@ -42,8 +45,11 @@ public String getThumbnailUrl() throws ParsingException {
 
                 // First look for track artwork url
                 if (trackObject.isString(ARTWORK_URL_KEY)) {
-                    final String url = trackObject.getString(ARTWORK_URL_KEY, "");
-                    if (!url.isEmpty()) return url;
+                    String artworkUrl = trackObject.getString(ARTWORK_URL_KEY, "");
+                    if (!artworkUrl.isEmpty()) {
+                        String artworkUrlBetterResolution = artworkUrl.replace("large.jpg", "crop.jpg");
+                        return artworkUrlBetterResolution;
+                    }
                 }
 
                 // Then look for track creator avatar url

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java

@@ -95,6 +95,7 @@ public KioskExtractor createNewKiosk(StreamingService streamingService,
         try {
             list.addKioskEntry(chartsFactory, h, "Top 50");
             list.addKioskEntry(chartsFactory, h, "New & hot");
+            list.setDefaultKiosk("New & hot");
         } catch (Exception e) {
             throw new ExtractionException(e);
         }

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java

@@ -57,7 +57,12 @@ public String getUploadDate() throws ParsingException {
     @Nonnull
     @Override
     public String getThumbnailUrl() {
-        return track.getString("artwork_url", "");
+        String artworkUrl = track.getString("artwork_url", "");
+        if (artworkUrl.isEmpty()) {
+            artworkUrl = track.getObject("user").getString("avatar_url", "");
+        }
+        String artworkUrlBetterResolution = artworkUrl.replace("large.jpg", "crop.jpg");
+        return artworkUrlBetterResolution;
     }
 
     @Nonnull

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamInfoItemExtractor.java

@@ -52,7 +52,12 @@ public long getViewCount() {
 
     @Override
     public String getThumbnailUrl() {
-        return itemObject.getString("artwork_url");
+        String artworkUrl = itemObject.getString("artwork_url", "");
+        if (artworkUrl.isEmpty()) {
+            artworkUrl = itemObject.getObject("user").getString("avatar_url");
+        }
+        String artworkUrlBetterResolution = artworkUrl.replace("large.jpg", "crop.jpg");
+        return artworkUrlBetterResolution;
     }
 
     @Override

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsExtractor.java

@@ -107,11 +107,11 @@ public InfoItemsPage<CommentsInfoItem> getPage(String pageUrl) throws IOExceptio
             throw new ParsingException("Could not parse json data for comments", e);
         }
         CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId());
-        collectCommentsFrom(collector, ajaxJson, pageUrl);
+        collectCommentsFrom(collector, ajaxJson);
         return new InfoItemsPage<>(collector, getNextPageUrl(ajaxJson));
     }
 
-    private void collectCommentsFrom(CommentsInfoItemsCollector collector, JsonObject ajaxJson, String pageUrl) throws ParsingException {
+    private void collectCommentsFrom(CommentsInfoItemsCollector collector, JsonObject ajaxJson) throws ParsingException {
 
         JsonArray contents;
         try {
@@ -130,7 +130,7 @@ private void collectCommentsFrom(CommentsInfoItemsCollector collector, JsonObjec
 
         for(Object c: comments) {
             if(c instanceof JsonObject) {
-                CommentsInfoItemExtractor extractor = new YoutubeCommentsInfoItemExtractor((JsonObject) c, pageUrl);
+                CommentsInfoItemExtractor extractor = new YoutubeCommentsInfoItemExtractor((JsonObject) c, getUrl());
                 collector.commit(extractor);
             }
         }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeCommentsInfoItemExtractor.java

@@ -38,7 +38,7 @@ public String getName() throws ParsingException {
         try {
             return YoutubeCommentsExtractor.getYoutubeText(JsonUtils.getObject(json, "authorText"));
         } catch (Exception e) {
-            throw new ParsingException("Could not get author name", e);
+            return "";
         }
     }
 
@@ -95,7 +95,7 @@ public String getAuthorName() throws ParsingException {
         try {
             return YoutubeCommentsExtractor.getYoutubeText(JsonUtils.getObject(json, "authorText"));
         } catch (Exception e) {
-            throw new ParsingException("Could not get author name", e);
+            return "";
         }
     }
 
@@ -104,7 +104,7 @@ public String getAuthorEndpoint() throws ParsingException {
         try {
             return "https://youtube.com/channel/" + JsonUtils.getString(json, "authorEndpoint.browseEndpoint.browseId");
         } catch (Exception e) {
-            throw new ParsingException("Could not get author endpoint", e);
+            return "";
         }
     }
 

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java

@@ -166,8 +166,8 @@ private String parseHtmlAndGetFullLinks(String descriptionHtml)
             throws MalformedURLException, UnsupportedEncodingException, ParsingException {
         final Document description = Jsoup.parse(descriptionHtml, getUrl());
         for(Element a : description.select("a")) {
-            final URL redirectLink = new URL(
-                    a.attr("abs:href"));
+            final String rawUrl = a.attr("abs:href");
+            final URL redirectLink = new URL(rawUrl);
             final String queryString = redirectLink.getQuery();
             if(queryString != null) {
                 // if the query string is null we are not dealing with a redirect link,
@@ -179,11 +179,15 @@ private String parseHtmlAndGetFullLinks(String descriptionHtml)
                     // if link is null the a tag is a hashtag.
                     // They refer to the youtube search. We do not handle them.
                     a.text(link);
+                    a.attr("href", link);
                 } else if(redirectLink.toString().contains("https://www.youtube.com/")) {
                     a.text(redirectLink.toString());
+                    a.attr("href", redirectLink.toString());
                 }
             } else if(redirectLink.toString().contains("https://www.youtube.com/")) {
+                descriptionHtml = descriptionHtml.replace(rawUrl, redirectLink.toString());
                 a.text(redirectLink.toString());
+                a.attr("href", redirectLink.toString());
             }
         }
         return description.select("body").first().html();
@@ -206,29 +210,40 @@ public int getAgeLimit() throws ParsingException {
     @Override
     public long getLength() throws ParsingException {
         assertPageFetched();
-        if(playerArgs != null) {
-            try {
-                long returnValue = Long.parseLong(playerArgs.get("length_seconds") + "");
-                if (returnValue >= 0) return returnValue;
-            } catch (Exception ignored) {
-                // Try other method...
+
+        final JsonObject playerResponse;
+        try {
+            final String pr;
+            if(playerArgs != null) {
+                pr = playerArgs.getString("player_response");
+            } else {
+                pr = videoInfoPage.get("player_response");
             }
+            playerResponse = JsonParser.object()
+                    .from(pr);
+        } catch (Exception e) {
+            throw new ParsingException("Could not get playerResponse", e);
         }
 
-        String lengthString = videoInfoPage.get("length_seconds");
+        // try getting duration from playerargs
         try {
-            return Long.parseLong(lengthString);
-        } catch (Exception ignored) {
-            // Try other method...
+            String durationMs = playerResponse
+                    .getObject("streamingData")
+                    .getArray("formats")
+                    .getObject(0)
+                    .getString("approxDurationMs");
+            return Long.parseLong(durationMs)/1000;
+        } catch (Exception e) {
         }
 
-        // TODO: 25.11.17 Implement a way to get the length for age restricted videos #44
+        //try getting value from age gated video
         try {
-            // Fallback to HTML method
-            return Long.parseLong(doc.select("div[class~=\"ytp-progress-bar\"][role=\"slider\"]").first()
-                    .attr("aria-valuemax"));
+            String duration = playerResponse
+                    .getObject("videoDetails")
+                    .getString("lengthSeconds");
+            return Long.parseLong(duration);
         } catch (Exception e) {
-            throw new ParsingException("Could not get video length", e);
+            throw new ParsingException("Every methode to get the duration has failed: ", e);
         }
     }
 
@@ -597,6 +612,7 @@ public void onFetchPage(@Nonnull Downloader downloader) throws IOException, Extr
         final String playerUrl;
         // Check if the video is age restricted
         if (pageContent.contains("<meta property=\"og:restrictions:age")) {
+            // do this if it is age gated
             final EmbeddedInfo info = getEmbeddedInfo();
             final String videoInfoUrl = getVideoInfoUrl(getId(), info.sts);
             final String infoPageResponse = downloader.download(videoInfoUrl);
@@ -685,11 +701,16 @@ private EmbeddedInfo getEmbeddedInfo() throws ParsingException, ReCaptchaExcepti
                 playerUrl = HTTPS + playerUrl;
             }
 
-            // Get embed sts
-            final String stsPattern = "\"sts\"\\s*:\\s*(\\d+)";
-            final String sts = Parser.matchGroup1(stsPattern, embedPageContent);
+            try {
+                // Get embed sts
+                final String stsPattern = "\"sts\"\\s*:\\s*(\\d+)";
+                final String sts = Parser.matchGroup1(stsPattern, embedPageContent);
+                return new EmbeddedInfo(playerUrl, sts);
+            } catch (Exception i) {
+                // if it failes we simply reply with no sts as then it does not seem to be necessary
+                return new EmbeddedInfo(playerUrl, "");
+            }
 
-            return new EmbeddedInfo(playerUrl, sts);
         } catch (IOException e) {
             throw new ParsingException(
                     "Could load decryption code form restricted video for the Youtube service.", e);
@@ -708,15 +729,8 @@ private String loadDecryptionCode(String playerUrl) throws DecryptException {
             }
 
             final String playerCode = downloader.download(playerUrl);
+            final String decryptionFunctionName = getDecryptionFuncName(playerCode);
 
-            final String decryptionFunctionName;
-            if (Parser.isMatch(DECRYPTION_AKAMAIZED_SHORT_STRING_REGEX, playerCode)) {
-                decryptionFunctionName = Parser.matchGroup1(DECRYPTION_AKAMAIZED_SHORT_STRING_REGEX, playerCode);
-            } else if (Parser.isMatch(DECRYPTION_AKAMAIZED_STRING_REGEX, playerCode)) {
-                decryptionFunctionName = Parser.matchGroup1(DECRYPTION_AKAMAIZED_STRING_REGEX, playerCode);
-            } else {
-                decryptionFunctionName = Parser.matchGroup1(DECYRYPTION_SIGNATURE_FUNCTION_REGEX, playerCode);
-            }
             final String functionPattern = "("
                     + decryptionFunctionName.replace("$", "\\$")
                     + "=function\\([a-zA-Z0-9_]+\\)\\{.+?\\})";
@@ -757,6 +771,27 @@ private String decryptSignature(String encryptedSig, String decryptionCode) thro
         return result == null ? "" : result.toString();
     }
 
+    private String getDecryptionFuncName(String playerCode) throws DecryptException {
+        String decryptionFunctionName;
+        // Cascading things in catch is ugly, but its faster than running a match before getting the actual name
+        // to se if the function can actually be found with the given regex.
+        // However if this cascading should propably be cleaned up somehow as it looks a bit weird.
+        try {
+            decryptionFunctionName = Parser.matchGroup1(DECYRYPTION_SIGNATURE_FUNCTION_REGEX, playerCode);
+        } catch (Parser.RegexException re) {
+            try {
+                decryptionFunctionName = Parser.matchGroup1(DECRYPTION_AKAMAIZED_SHORT_STRING_REGEX, playerCode);
+            } catch (Parser.RegexException re2) {
+                try {
+                    decryptionFunctionName = Parser.matchGroup1(DECRYPTION_AKAMAIZED_STRING_REGEX, playerCode);
+                } catch (Parser.RegexException re3) {
+                    throw new DecryptException("Could not find decrypt function with any of the given patterns.", re);
+                }
+            }
+        }
+        return decryptionFunctionName;
+    }
+
     @Nonnull
     private List<SubtitlesInfo> getAvailableSubtitlesInfo() throws SubtitlesException {
         // If the video is age restricted getPlayerConfig will fail
@@ -874,7 +909,13 @@ private Map<String, ItagItem> getItags(String encodedUrlMapKey, ItagItem.ItagTyp
                         String streamUrl = tags.get("url");
                         // if video has a signature: decrypt it and add it to the url
                         if (tags.get("s") != null) {
-                            streamUrl = streamUrl + "&signature=" + decryptSignature(tags.get("s"), decryptionCode);
+                            if (tags.get("sp") == null) {
+                                // fallback for urls not conaining the "sp" tag
+                                streamUrl = streamUrl + "&signature=" + decryptSignature(tags.get("s"), decryptionCode);
+                            }
+                            else {
+                                streamUrl = streamUrl + "&" + tags.get("sp") + "=" + decryptSignature(tags.get("s"), decryptionCode);
+                            }
                         }
                         urlAndItags.put(streamUrl, itagItem);
                     }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSubscriptionExtractor.java

@@ -63,16 +63,10 @@ private List<SubscriptionItem> getItemsFromOPML(InputStream contentInputStream)
             String title = outline.attr("title");
             String xmlUrl = outline.attr("abs:xmlUrl");
 
-            if (title.isEmpty() || xmlUrl.isEmpty()) {
-                throw new InvalidSourceException("document has invalid entries");
-            }
-
             try {
                 String id = Parser.matchGroup1(ID_PATTERN, xmlUrl);
                 result.add(new SubscriptionItem(service.getServiceId(), BASE_CHANNEL_URL + id, title));
-            } catch (Parser.RegexException e) {
-                throw new InvalidSourceException("document has invalid entries", e);
-            }
+            } catch (Parser.RegexException ignored) { /* ignore invalid subscriptions */ }
         }
 
         return result;

extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCOggTest.java

@@ -36,7 +36,7 @@ public void getAudioStreamsCount() throws Exception {
     @Test
     public void getAudioStreamsContainOgg() throws Exception {
         for(AudioStream stream : extractor.getAudioStreams()) {
-            System.out.println(stream.getFormat());
+            assertEquals("OGG", stream.getFormat().toString());
         }
     }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractorTest.java

@@ -36,7 +36,6 @@ public void testGetDownloader() throws Exception {
         assertNotNull(NewPipe.getDownloader());
     }
 
-    @Ignore
     @Test
     public void testGetName() throws Exception {
         assertEquals(extractor.getName(), "Top 50");

extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java

@@ -226,6 +226,7 @@ public static void setUp() throws Exception {
         // Additional Testing
         //////////////////////////////////////////////////////////////////////////*/
 
+        @Ignore
         @Test
         public void testGetPageInNewExtractor() throws Exception {
             final PlaylistExtractor newExtractor = SoundCloud.getPlaylistExtractor(extractor.getUrl());
@@ -270,6 +271,8 @@ public void testRelatedItems() throws Exception {
             defaultTestRelatedItems(extractor, SoundCloud.getServiceId());
         }
 
+        //TODO: FUCK THIS: This triggers a 500 at sever
+        @Ignore
         @Test
         public void testMoreRelatedItems() throws Exception {
             ListExtractor.InfoItemsPage<StreamInfoItem> currentPage = defaultTestMoreItems(extractor, ServiceList.SoundCloud.getServiceId());