From a281519e771d139e7d6cd1b7f64dece92cbb5124 Mon Sep 17 00:00:00 2001 From: bopol Date: Sun, 19 Jan 2020 12:45:52 +0100 Subject: [PATCH 01/18] added metadata, fix descriptions, fix thumbnail, update tests thumbnail: quality before: https://peertube.cpy.re/static/thumbnails/d2a5ec78-5f85-4090-8ec5-dc1102e022ea.jpg quality after: https://peertube.cpy.re/static/previews/d2a5ec78-5f85-4090-8ec5-dc1102e022ea.jpg description: we were getting about the first 260 characters, we now get full description (with fallback to first 260 chars if the get request for full description fails) test: updated tests to match description, also changed some test: it was assertEquals(extracted, expected), but the proper way to do it is assertEquals(expected, extracted) metadata: got host, privacy (public, private, unlisted), licence, language, tags --- README.md | 2 +- .../extractors/MediaCCCStreamExtractor.java | 32 +++++++ .../extractors/PeertubeStreamExtractor.java | 50 +++++++++-- .../soundcloud/SoundcloudStreamExtractor.java | 31 +++++++ .../extractors/YoutubeStreamExtractor.java | 31 +++++++ .../extractor/stream/StreamExtractor.java | 57 ++++++++++++ .../newpipe/extractor/stream/StreamInfo.java | 88 +++++++++++++++++++ .../schabi/newpipe/extractor/utils/Utils.java | 1 - .../PeertubeCommentsExtractorTest.java | 4 +- .../PeertubeStreamExtractorDefaultTest.java | 21 ++--- 10 files changed, 295 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index f8c9c8cf2a..66515055de 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ To test changes quickly you can build the library locally. Using the local Maven 2. It's _recommended_ that you change the `version` of this library (e.g. `LOCAL_SNAPSHOT`). 3. Run gradle's `ìnstall` task to deploy this library to your local repository (using the wrapper, present in the root of this project: `./gradlew install`) 4. Change the dependency version used in your project to match the one you chose in step 2 (`implementation 'com.github.TeamNewPipe:NewPipeExtractor:LOCAL_SNAPSHOT'`) - + > Tip for Android Studio users: After you make changes and run the `install` task, use the menu option `File → "Sync with File System"` to refresh the library in your project. ## Supported sites diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java index a1dfd2454c..84fb76859c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java @@ -12,6 +12,7 @@ import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.stream.*; +import org.schabi.newpipe.extractor.utils.JsonUtils; import javax.annotation.Nonnull; import java.io.IOException; @@ -225,4 +226,35 @@ public String getName() throws ParsingException { public String getOriginalUrl() throws ParsingException { return data.getString("frontend_link"); } + + @Override + public String getHost() throws ParsingException { + return ""; + } + + @Override + public String getPrivacy() throws ParsingException { + return ""; + } + + @Override + public String getCategory() throws ParsingException { + return ""; + } + + @Override + public String getLicence() throws ParsingException { + return ""; + } + + @Override + public String getStreamInfoLanguage() throws ParsingException { + return ""; + } + + @Nonnull + @Override + public List getTags() throws ParsingException { + return new ArrayList<>(); + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index d8d0de0059..0bead848ca 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -9,7 +9,7 @@ import org.jsoup.helper.StringUtil; import org.schabi.newpipe.extractor.MediaFormat; -import org.schabi.newpipe.extractor.ServiceList; +import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.downloader.Response; @@ -29,7 +29,6 @@ import org.schabi.newpipe.extractor.stream.SubtitlesStream; import org.schabi.newpipe.extractor.stream.VideoStream; import org.schabi.newpipe.extractor.utils.JsonUtils; -import org.schabi.newpipe.extractor.utils.Utils; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; @@ -66,16 +65,25 @@ public DateWrapper getUploadDate() throws ParsingException { @Override public String getThumbnailUrl() throws ParsingException { - return baseUrl + JsonUtils.getString(json, "thumbnailPath"); + return baseUrl + JsonUtils.getString(json, "previewPath"); } @Override public String getDescription() throws ParsingException { + String description = ""; + Downloader dl = NewPipe.getDownloader(); try { - return JsonUtils.getString(json, "description"); - }catch(ParsingException e) { - return "No description"; + Response response = dl.get(getUrl() + "/description"); + JsonObject jsonObject = JsonParser.object().from(response.responseBody()); + description = JsonUtils.getString(jsonObject, "description"); + } catch (ReCaptchaException | IOException | JsonParserException e) { + e.printStackTrace(); } + if (description.equals("")) { + //if the request above failed + description = JsonUtils.getString(json, "description"); + } + return description; } @Override @@ -224,8 +232,9 @@ public StreamInfoItemsCollector getRelatedStreams() throws IOException, Extracti if(!StringUtil.isBlank(apiUrl)) getStreamsFromApi(collector, apiUrl); return collector; } - - private List getTags(){ + + @Override + public List getTags(){ try { return (List) JsonUtils.getArray(json, "tags"); } catch (Exception e) { @@ -339,4 +348,29 @@ public String getOriginalUrl() throws ParsingException { return baseUrl + "/videos/watch/" + getId(); } + //TODO: change privacy, category, licence by getting ID, therefore we will be able to translate it + @Override + public String getHost() throws ParsingException { + return JsonUtils.getString(json, "account.host"); + } + + @Override + public String getPrivacy() throws ParsingException { + return JsonUtils.getString(json, "privacy.label"); + } + + @Override + public String getCategory() throws ParsingException { + return JsonUtils.getString(json, "category.label"); + } + + @Override + public String getLicence() throws ParsingException { + return JsonUtils.getString(json, "licence.label"); + } + + @Override + public String getStreamInfoLanguage() throws ParsingException { + return JsonUtils.getString(json, "language.label"); + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java index 14f7023f49..19ab987cde 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java @@ -254,4 +254,35 @@ public StreamInfoItemsCollector getRelatedStreams() throws IOException, Extracti public String getErrorMessage() { return null; } + + @Override + public String getHost() throws ParsingException { + return ""; + } + + @Override + public String getPrivacy() throws ParsingException { + return ""; + } + + @Override + public String getCategory() throws ParsingException { + return ""; + } + + @Override + public String getLicence() throws ParsingException { + return ""; + } + + @Override + public String getStreamInfoLanguage() throws ParsingException { + return ""; + } + + @Nonnull + @Override + public List getTags() throws ParsingException { + return new ArrayList<>(); + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index c05004ede7..0e380fbce1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -1113,4 +1113,35 @@ public List getFrames() throws ExtractionException { throw new ExtractionException(e); } } + + @Override + public String getHost() throws ParsingException { + return ""; + } + + @Override + public String getPrivacy() throws ParsingException { + return ""; + } + + @Override + public String getCategory() throws ParsingException { + return ""; + } + + @Override + public String getLicence() throws ParsingException { + return ""; + } + + @Override + public String getStreamInfoLanguage() throws ParsingException { + return ""; + } + + @Nonnull + @Override + public List getTags() throws ParsingException { + return new ArrayList<>(); + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java index d697dff2ca..ce18dfa3b0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java @@ -27,6 +27,7 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.localization.DateWrapper; +import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Parser; import javax.annotation.Nonnull; @@ -349,4 +350,60 @@ protected long getTimestampSeconds(String regexPattern) throws ParsingException return 0; } } + + /** + * The host of the stream (Eg. peertube.cpy.re). + * If the privacy is not available, or if the service doesn't use + * a federated system, but a centralised system, + * you can simply return an empty string. + * @return the host of the stream or an empty String. + * @throws ParsingException + */ + @Nonnull + public abstract String getHost() throws ParsingException; + + /** + * The privacy of the stream (Eg. Public, Private, Unlisted…). + * If the privacy is not available you can simply return an empty string. + * @return the privacy of the stream or an empty String. + * @throws ParsingException + */ + @Nonnull + public abstract String getPrivacy() throws ParsingException; + + /** + * The name of the category of the stream. + * If the category is not available you can simply return an empty string. + * @return the category of the stream or an empty String. + * @throws ParsingException + */ + @Nonnull + public abstract String getCategory() throws ParsingException; + + /** + * The name of the licence of the stream. + * If the licence is not available you can simply return an empty string. + * @return the licence of the stream or an empty String. + * @throws ParsingException + */ + @Nonnull + public abstract String getLicence() throws ParsingException; + + /** + * The language of the stream. + * If the language is not available you can simply return an empty string. + * @return the licence of the stream or an empty String. + * @throws ParsingException + */ + @Nonnull + public abstract String getStreamInfoLanguage() throws ParsingException; + + /** + * The list of tags of the stream. + * If the tag list is not available you can simply return an empty list. + * @return the list of tags of the stream or an empty list. + * @throws ParsingException + */ + @Nonnull + public abstract List getTags() throws ParsingException; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java index 34ea703c2c..111f387d74 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java @@ -270,6 +270,38 @@ private static StreamInfo extractOptionalData(StreamInfo streamInfo, StreamExtra streamInfo.addError(e); } + //additional info + try { + streamInfo.setHost(extractor.getHost()); + } catch (Exception e) { + streamInfo.addError(e); + } + try { + streamInfo.setPrivacy(extractor.getPrivacy()); + } catch (Exception e) { + streamInfo.addError(e); + } + try { + streamInfo.setCategory(extractor.getCategory()); + } catch (Exception e) { + streamInfo.addError(e); + } + try { + streamInfo.setLicence(extractor.getLicence()); + } catch (Exception e) { + streamInfo.addError(e); + } + try { + streamInfo.setLanguage(extractor.getStreamInfoLanguage()); + } catch (Exception e) { + streamInfo.addError(e); + } + try { + streamInfo.setTags(extractor.getTags()); + } catch (Exception e) { + streamInfo.addError(e); + } + streamInfo.setRelatedStreams(ExtractorHelper.getRelatedVideosOrLogError(streamInfo, extractor)); return streamInfo; @@ -308,6 +340,13 @@ private static StreamInfo extractOptionalData(StreamInfo streamInfo, StreamExtra private long startPosition = 0; private List subtitles = new ArrayList<>(); + private String host = ""; + private String privacy = ""; + private String category = ""; + private String licence = ""; + private String language = ""; + private List tags = new ArrayList<>(); + /** * Get the stream type * @@ -533,4 +572,53 @@ public void setSubtitles(List subtitles) { this.subtitles = subtitles; } + public String getHost() { + return this.host; + } + + public void setHost(String str) { + this.host = str; + } + + public String getPrivacy() { + return this.privacy; + } + + public void setPrivacy(String str) { + this.privacy = str; + } + + public String getCategory() { + return this.category; + } + + public void setCategory(String cat) { + this.category = cat; + } + + public String getLicence() { + return this.licence; + } + + public void setLicence(String str) { + this.licence = str; + } + + public String getLanguage() { + return this.language; + } + + public void setLanguage(String lang) { + this.language = lang; + } + + public List getTags() { + return this.tags; + } + + public void setTags(List tags) { + this.tags = tags; + } + + } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java index fd06d40f58..3489b6d603 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java @@ -185,5 +185,4 @@ public static String getBaseUrl(String url) throws ParsingException { } return uri.getProtocol() + "://" + uri.getAuthority(); } - } \ No newline at end of file diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java index fef1c3ee52..e0a1d18931 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java @@ -26,7 +26,7 @@ public class PeertubeCommentsExtractorTest { public static void setUp() throws Exception { NewPipe.init(DownloaderTestImpl.getInstance()); extractor = (PeertubeCommentsExtractor) PeerTube - .getCommentsExtractor("https://peertube.mastodon.host/videos/watch/04af977f-4201-4697-be67-a8d8cae6fa7a"); + .getCommentsExtractor("https://framatube.org/videos/watch/04af977f-4201-4697-be67-a8d8cae6fa7a"); } @Test @@ -46,7 +46,7 @@ public void testGetComments() throws IOException, ExtractionException { @Test public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException { boolean result = false; - CommentsInfo commentsInfo = CommentsInfo.getInfo("https://peertube.mastodon.host/videos/watch/a8ea95b8-0396-49a6-8f30-e25e25fb2828"); + CommentsInfo commentsInfo = CommentsInfo.getInfo("https://framatube.org/videos/watch/a8ea95b8-0396-49a6-8f30-e25e25fb2828"); assertTrue("Comments".equals(commentsInfo.getName())); result = findInComments(commentsInfo.getRelatedItems(), "Loved it!!!"); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java index 75a692d422..b2175e851c 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java @@ -29,13 +29,14 @@ */ public class PeertubeStreamExtractorDefaultTest { private static PeertubeStreamExtractor extractor; + private static final String expectedDescription = "**[Want to help to translate this video?](https://weblate.framasoft.org/projects/what-is-peertube-video/)**\r\n\r\n**Take back the control of your videos! [#JoinPeertube](https://joinpeertube.org)**\r\n*A decentralized video hosting network, based on free/libre software!*\r\n\r\n**Animation Produced by:** [LILA](https://libreart.info) - [ZeMarmot Team](https://film.zemarmot.net)\r\n*Directed by* Aryeom\r\n*Assistant* Jehan\r\n**Licence**: [CC-By-SA 4.0](https://creativecommons.org/licenses/by-sa/4.0/)\r\n\r\n**Sponsored by** [Framasoft](https://framasoft.org)\r\n\r\n**Music**: [Red Step Forward](http://play.dogmazic.net/song.php?song_id=52491) - CC-By Ken Bushima\r\n\r\n**Movie Clip**: [Caminades 3: Llamigos](http://www.caminandes.com/) CC-By Blender Institute\r\n\r\n**Video sources**: https://gitlab.gnome.org/Jehan/what-is-peertube/"; @BeforeClass public static void setUp() throws Exception { NewPipe.init(DownloaderTestImpl.getInstance()); // setting instance might break test when running in parallel - PeerTube.setInstance(new PeertubeInstance("https://peertube.mastodon.host", "PeerTube on Mastodon.host")); - extractor = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://peertube.mastodon.host/videos/watch/afe5bf12-c58b-4efd-b56e-29c5a59e04bc"); + PeerTube.setInstance(new PeertubeInstance("https://framatube.org", "FramaTube")); + extractor = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://framatube.org/videos/watch/9c9de5e8-0a1e-484a-b099-e80766180a6d"); extractor.fetchPage(); } @@ -47,22 +48,22 @@ public void testGetInvalidTimeStamp() throws ParsingException { @Test public void testGetTitle() throws ParsingException { - assertEquals(extractor.getName(), "Power Corrupts the Best"); + assertEquals("What is PeerTube?", extractor.getName()); } @Test public void testGetDescription() throws ParsingException { - assertEquals(extractor.getDescription(), "A short reading from Bakunin, made for the group Audible Anarchist https://audibleanarchist.github.io/Webpage/"); + assertEquals(expectedDescription, extractor.getDescription()); } @Test public void testGetUploaderName() throws ParsingException { - assertEquals(extractor.getUploaderName(), "Rowsedower"); + assertEquals("Framasoft", extractor.getUploaderName()); } @Test public void testGetLength() throws ParsingException { - assertEquals(extractor.getLength(), 269); + assertEquals(113, extractor.getLength()); } @Test @@ -74,7 +75,7 @@ public void testGetViewCount() throws ParsingException { @Test public void testGetUploadDate() throws ParsingException, ParseException { final Calendar instance = Calendar.getInstance(); - instance.setTime(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'").parse("2018-09-30T14:08:24.378Z")); + instance.setTime(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'").parse("2018-10-01T10:52:46.396Z")); assertEquals(instance, requireNonNull(extractor.getUploadDate()).date()); } @@ -82,7 +83,7 @@ public void testGetUploadDate() throws ParsingException, ParseException { @Test public void testGetUploaderUrl() throws ParsingException { assertIsSecureUrl(extractor.getUploaderUrl()); - assertEquals("https://peertube.mastodon.host/api/v1/accounts/reddebrek@peertube.mastodon.host", extractor.getUploaderUrl()); + assertEquals("https://framatube.org/api/v1/accounts/framasoft@framatube.org", extractor.getUploaderUrl()); } @Test @@ -115,11 +116,11 @@ public void testGetRelatedVideos() throws ExtractionException, IOException { @Test public void testGetSubtitlesListDefault() throws IOException, ExtractionException { - assertTrue(extractor.getSubtitlesDefault().isEmpty()); + assertFalse(extractor.getSubtitlesDefault().isEmpty()); } @Test public void testGetSubtitlesList() throws IOException, ExtractionException { - assertTrue(extractor.getSubtitlesDefault().isEmpty()); + assertFalse(extractor.getSubtitlesDefault().isEmpty()); } } From b382416372add1c9420cf1657f67c9cae825751f Mon Sep 17 00:00:00 2001 From: bopol Date: Mon, 20 Jan 2020 14:36:12 +0100 Subject: [PATCH 02/18] changed the way to extract peertube description if the description length is above 254, and ends with ..., it means the description we got from the first request is shortened. why above 254: because in fact, shortened descriptions length are between 255 : https://framatube.org/videos/watch/24efbe1b-73c0-4d72-a3ff-77c8b32d3fcd https://framatube.org/videos/watch/1ca99f40-fb5b-4fa4-abe5-4d54325df7fc and 269: https://framatube.org/videos/watch/4d661d5f-a884-4510-bca8-15cb19aa3fe5 also fixed a typo in StreamExtractor.java --- .../extractors/PeertubeStreamExtractor.java | 25 +++++++++++-------- .../extractor/stream/StreamExtractor.java | 2 +- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index 0bead848ca..ac3d6ecbed 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -70,20 +70,23 @@ public String getThumbnailUrl() throws ParsingException { @Override public String getDescription() throws ParsingException { - String description = ""; - Downloader dl = NewPipe.getDownloader(); + String desc; try { - Response response = dl.get(getUrl() + "/description"); - JsonObject jsonObject = JsonParser.object().from(response.responseBody()); - description = JsonUtils.getString(jsonObject, "description"); - } catch (ReCaptchaException | IOException | JsonParserException e) { - e.printStackTrace(); + desc = JsonUtils.getString(json, "description"); + } catch (ParsingException e) { + return "No description"; } - if (description.equals("")) { - //if the request above failed - description = JsonUtils.getString(json, "description"); + if (desc.length() >= 255 && desc.substring(desc.length() - 3).equals("...")) { + Downloader dl = NewPipe.getDownloader(); + try { + Response response = dl.get(getUrl() + "/description"); + JsonObject jsonObject = JsonParser.object().from(response.responseBody()); + desc = JsonUtils.getString(jsonObject, "description"); + } catch (ReCaptchaException | IOException | JsonParserException e) { + e.printStackTrace(); + } } - return description; + return desc; } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java index ce18dfa3b0..a638c7b6d1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java @@ -353,7 +353,7 @@ protected long getTimestampSeconds(String regexPattern) throws ParsingException /** * The host of the stream (Eg. peertube.cpy.re). - * If the privacy is not available, or if the service doesn't use + * If the host is not available, or if the service doesn't use * a federated system, but a centralised system, * you can simply return an empty string. * @return the host of the stream or an empty String. From ad7f97ae836f89a0f555e1062e5cc3b5211b1dc3 Mon Sep 17 00:00:00 2001 From: bopol Date: Mon, 20 Jan 2020 15:14:23 +0100 Subject: [PATCH 03/18] fix PeerTube description and add more description tests actually, the max description length is 250 after request with our extractor. during my tests, I made API requests with Firefox, copy/pasted into echo "insert description" | wc, and it was giving a wrong length, maybe due to the escapers, I have no idea anyway, it's now fixed --- .../extractors/PeertubeStreamExtractor.java | 3 ++- .../PeertubeStreamExtractorDefaultTest.java | 22 ++++++++++++++++--- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index ac3d6ecbed..c922b2fef5 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -76,7 +76,8 @@ public String getDescription() throws ParsingException { } catch (ParsingException e) { return "No description"; } - if (desc.length() >= 255 && desc.substring(desc.length() - 3).equals("...")) { + if (desc.length() == 250 && desc.substring(desc.length() - 3).equals("...")) { + //if description is shortened, get full description Downloader dl = NewPipe.getDownloader(); try { Response response = dl.get(getUrl() + "/description"); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java index b2175e851c..2a4378a207 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java @@ -29,7 +29,8 @@ */ public class PeertubeStreamExtractorDefaultTest { private static PeertubeStreamExtractor extractor; - private static final String expectedDescription = "**[Want to help to translate this video?](https://weblate.framasoft.org/projects/what-is-peertube-video/)**\r\n\r\n**Take back the control of your videos! [#JoinPeertube](https://joinpeertube.org)**\r\n*A decentralized video hosting network, based on free/libre software!*\r\n\r\n**Animation Produced by:** [LILA](https://libreart.info) - [ZeMarmot Team](https://film.zemarmot.net)\r\n*Directed by* Aryeom\r\n*Assistant* Jehan\r\n**Licence**: [CC-By-SA 4.0](https://creativecommons.org/licenses/by-sa/4.0/)\r\n\r\n**Sponsored by** [Framasoft](https://framasoft.org)\r\n\r\n**Music**: [Red Step Forward](http://play.dogmazic.net/song.php?song_id=52491) - CC-By Ken Bushima\r\n\r\n**Movie Clip**: [Caminades 3: Llamigos](http://www.caminandes.com/) CC-By Blender Institute\r\n\r\n**Video sources**: https://gitlab.gnome.org/Jehan/what-is-peertube/"; + private static final String expectedLargeDescription = "**[Want to help to translate this video?](https://weblate.framasoft.org/projects/what-is-peertube-video/)**\r\n\r\n**Take back the control of your videos! [#JoinPeertube](https://joinpeertube.org)**\r\n*A decentralized video hosting network, based on free/libre software!*\r\n\r\n**Animation Produced by:** [LILA](https://libreart.info) - [ZeMarmot Team](https://film.zemarmot.net)\r\n*Directed by* Aryeom\r\n*Assistant* Jehan\r\n**Licence**: [CC-By-SA 4.0](https://creativecommons.org/licenses/by-sa/4.0/)\r\n\r\n**Sponsored by** [Framasoft](https://framasoft.org)\r\n\r\n**Music**: [Red Step Forward](http://play.dogmazic.net/song.php?song_id=52491) - CC-By Ken Bushima\r\n\r\n**Movie Clip**: [Caminades 3: Llamigos](http://www.caminandes.com/) CC-By Blender Institute\r\n\r\n**Video sources**: https://gitlab.gnome.org/Jehan/what-is-peertube/"; + private static final String expectedSmallDescription = "https://www.kickstarter.com/projects/1587081065/nothing-to-hide-the-documentary"; @BeforeClass public static void setUp() throws Exception { @@ -52,8 +53,23 @@ public void testGetTitle() throws ParsingException { } @Test - public void testGetDescription() throws ParsingException { - assertEquals(expectedDescription, extractor.getDescription()); + public void testGetLargeDescription() throws ParsingException { + assertEquals(expectedLargeDescription, extractor.getDescription()); + } + + @Test + public void testGetEmptyDescription() throws Exception { + PeertubeStreamExtractor extractorEmpty = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://framatube.org/api/v1/videos/d5907aad-2252-4207-89ec-a4b687b9337d"); + extractorEmpty.fetchPage(); + assertEquals("No description", extractorEmpty.getDescription()); + } + + @Test + public void testGetSmallDescription() throws Exception { + PeerTube.setInstance(new PeertubeInstance("https://peertube.cpy.re", "PeerTube test server")); + PeertubeStreamExtractor extractorSmall = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://peertube.cpy.re/videos/watch/d2a5ec78-5f85-4090-8ec5-dc1102e022ea"); + extractorSmall.fetchPage(); + assertEquals(expectedSmallDescription, extractorSmall.getDescription()); } @Test From f403490bd9ada2b19176ffc694d5b5ad405975b6 Mon Sep 17 00:00:00 2001 From: B0pol Date: Mon, 20 Jan 2020 21:28:26 +0100 Subject: [PATCH 04/18] Refactoring Co-Authored-By: Tobias Groza --- .../services/peertube/extractors/PeertubeStreamExtractor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index c922b2fef5..0de72a9405 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -76,7 +76,7 @@ public String getDescription() throws ParsingException { } catch (ParsingException e) { return "No description"; } - if (desc.length() == 250 && desc.substring(desc.length() - 3).equals("...")) { + if (desc.length() == 250 && desc.substring(247).equals("...")) { //if description is shortened, get full description Downloader dl = NewPipe.getDownloader(); try { From 1a15c0e750dc95fa0a92cc1c988e781e66631fa7 Mon Sep 17 00:00:00 2001 From: B0pol Date: Thu, 23 Jan 2020 04:42:54 +0100 Subject: [PATCH 05/18] agelimit now returns 18 if the video is marked as nsfw, 0 otherwise + created getBoolean method in JsonUtils.java --- .../peertube/extractors/PeertubeStreamExtractor.java | 8 ++++++-- .../org/schabi/newpipe/extractor/utils/JsonUtils.java | 10 ++++++++++ .../peertube/PeertubeStreamExtractorDefaultTest.java | 8 ++++++++ 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index 0de72a9405..b841d9fd9a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -92,7 +92,12 @@ public String getDescription() throws ParsingException { @Override public int getAgeLimit() throws ParsingException { - return NO_AGE_LIMIT; + boolean isNSFW = JsonUtils.getBoolean(json, "nsfw"); + if (isNSFW) { + return 18; + } else { + return NO_AGE_LIMIT; + } } @Override @@ -352,7 +357,6 @@ public String getOriginalUrl() throws ParsingException { return baseUrl + "/videos/watch/" + getId(); } - //TODO: change privacy, category, licence by getting ID, therefore we will be able to translate it @Override public String getHost() throws ParsingException { return JsonUtils.getString(json, "account.host"); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java index e5d7bb62f6..ebd251d7ce 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java @@ -37,6 +37,16 @@ public static String getString(@Nonnull JsonObject object, @Nonnull String path) throw new ParsingException("Unable to get " + path); } } + + @Nonnull + public static Boolean getBoolean(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{ + Object value = getValue(object, path); + if(value instanceof Boolean) { + return (Boolean) value; + }else { + throw new ParsingException("Unable to get " + path); + } + } @Nonnull public static Number getNumber(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{ diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java index 2a4378a207..4166b192ff 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java @@ -139,4 +139,12 @@ public void testGetSubtitlesListDefault() throws IOException, ExtractionExceptio public void testGetSubtitlesList() throws IOException, ExtractionException { assertFalse(extractor.getSubtitlesDefault().isEmpty()); } + + @Test + public void testGetAgeLimit() throws ExtractionException, IOException { + assertEquals(0, extractor.getAgeLimit()); + PeertubeStreamExtractor ageLimit = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://peertube.co.uk/videos/watch/6762bb04-cad5-407b-81ee-c18eac4715a7"); + ageLimit.fetchPage(); + assertEquals(18, ageLimit.getAgeLimit()); + } } From 74439f692a234d89b5556e1949192f3f9681f840 Mon Sep 17 00:00:00 2001 From: bopol Date: Thu, 23 Jan 2020 14:19:22 +0100 Subject: [PATCH 06/18] add extraction for support info and rename getLanguageInfo function --- .../extractors/MediaCCCStreamExtractor.java | 8 +++++++- .../extractors/PeertubeStreamExtractor.java | 12 ++++++++++-- .../soundcloud/SoundcloudStreamExtractor.java | 8 +++++++- .../extractors/YoutubeStreamExtractor.java | 8 +++++++- .../extractor/stream/StreamExtractor.java | 14 +++++++++++++- .../newpipe/extractor/stream/StreamInfo.java | 18 +++++++++++++++--- .../PeertubeStreamExtractorDefaultTest.java | 7 +++++++ 7 files changed, 66 insertions(+), 9 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java index 84fb76859c..9168d6e61b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java @@ -248,7 +248,7 @@ public String getLicence() throws ParsingException { } @Override - public String getStreamInfoLanguage() throws ParsingException { + public String getLanguageInfo() throws ParsingException { return ""; } @@ -257,4 +257,10 @@ public String getStreamInfoLanguage() throws ParsingException { public List getTags() throws ParsingException { return new ArrayList<>(); } + + @Nonnull + @Override + public String getSupportInfo() throws ParsingException { + return ""; + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index b841d9fd9a..4dadae7785 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -35,6 +35,8 @@ import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; +import javax.annotation.Nonnull; + public class PeertubeStreamExtractor extends StreamExtractor { @@ -250,7 +252,13 @@ public List getTags(){ return Collections.emptyList(); } } - + + @Nonnull + @Override + public String getSupportInfo() throws ParsingException { + return JsonUtils.getString(json, "support"); + } + private String getRelatedStreamsUrl(List tags) throws UnsupportedEncodingException { String url = baseUrl + PeertubeSearchQueryHandlerFactory.SEARCH_ENDPOINT; StringBuilder params = new StringBuilder(); @@ -378,7 +386,7 @@ public String getLicence() throws ParsingException { } @Override - public String getStreamInfoLanguage() throws ParsingException { + public String getLanguageInfo() throws ParsingException { return JsonUtils.getString(json, "language.label"); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java index 19ab987cde..c5d34bd01c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java @@ -276,7 +276,7 @@ public String getLicence() throws ParsingException { } @Override - public String getStreamInfoLanguage() throws ParsingException { + public String getLanguageInfo() throws ParsingException { return ""; } @@ -285,4 +285,10 @@ public String getStreamInfoLanguage() throws ParsingException { public List getTags() throws ParsingException { return new ArrayList<>(); } + + @Nonnull + @Override + public String getSupportInfo() throws ParsingException { + return ""; + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index ea9a84d46a..74a44a6657 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -1156,7 +1156,7 @@ public String getLicence() throws ParsingException { } @Override - public String getStreamInfoLanguage() throws ParsingException { + public String getLanguageInfo() throws ParsingException { return ""; } @@ -1165,4 +1165,10 @@ public String getStreamInfoLanguage() throws ParsingException { public List getTags() throws ParsingException { return new ArrayList<>(); } + + @Nonnull + @Override + public String getSupportInfo() throws ParsingException { + return ""; + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java index a638c7b6d1..31a5a583b7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java @@ -396,7 +396,7 @@ protected long getTimestampSeconds(String regexPattern) throws ParsingException * @throws ParsingException */ @Nonnull - public abstract String getStreamInfoLanguage() throws ParsingException; + public abstract String getLanguageInfo() throws ParsingException; /** * The list of tags of the stream. @@ -406,4 +406,16 @@ protected long getTimestampSeconds(String regexPattern) throws ParsingException */ @Nonnull public abstract List getTags() throws ParsingException; + + /** + * The support information of the stream. + * see: https://framatube.org/videos/watch/ee408ec8-07cd-4e35-b884-fb681a4b9d37 + * (support button). + * If the support information are not available, + * you can simply return an empty String. + * @return the support information of the stream or an empty String. + * @throws ParsingException + */ + @Nonnull + public abstract String getSupportInfo() throws ParsingException; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java index 111f387d74..e32a133d07 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java @@ -292,7 +292,7 @@ private static StreamInfo extractOptionalData(StreamInfo streamInfo, StreamExtra streamInfo.addError(e); } try { - streamInfo.setLanguage(extractor.getStreamInfoLanguage()); + streamInfo.setLanguageInfo(extractor.getLanguageInfo()); } catch (Exception e) { streamInfo.addError(e); } @@ -301,6 +301,11 @@ private static StreamInfo extractOptionalData(StreamInfo streamInfo, StreamExtra } catch (Exception e) { streamInfo.addError(e); } + try { + streamInfo.setSupportInfo(extractor.getSupportInfo()); + } catch (Exception e) { + streamInfo.addError(e); + } streamInfo.setRelatedStreams(ExtractorHelper.getRelatedVideosOrLogError(streamInfo, extractor)); @@ -345,6 +350,7 @@ private static StreamInfo extractOptionalData(StreamInfo streamInfo, StreamExtra private String category = ""; private String licence = ""; private String language = ""; + private String support = ""; private List tags = new ArrayList<>(); /** @@ -604,11 +610,11 @@ public void setLicence(String str) { this.licence = str; } - public String getLanguage() { + public String getLanguageInfo() { return this.language; } - public void setLanguage(String lang) { + public void setLanguageInfo(String lang) { this.language = lang; } @@ -620,5 +626,11 @@ public void setTags(List tags) { this.tags = tags; } + public void setSupportInfo(String support) { + this.support = support; + } + public String getSupportInfo() { + return this.support; + } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java index 4166b192ff..8296785262 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java @@ -147,4 +147,11 @@ public void testGetAgeLimit() throws ExtractionException, IOException { ageLimit.fetchPage(); assertEquals(18, ageLimit.getAgeLimit()); } + + @Test + public void testGetSupportInformation() throws ExtractionException, IOException { + PeertubeStreamExtractor supportInfoExtractor = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://framatube.org/videos/watch/ee408ec8-07cd-4e35-b884-fb681a4b9d37"); + supportInfoExtractor.fetchPage(); + assertEquals("https://utip.io/chatsceptique", supportInfoExtractor.getSupportInfo()); + } } From 20da4750f8f779e7fd8efaf9471b890cd47c75a7 Mon Sep 17 00:00:00 2001 From: bopol Date: Thu, 23 Jan 2020 14:37:14 +0100 Subject: [PATCH 07/18] empty support returns "", same for empty description --- .../peertube/extractors/PeertubeStreamExtractor.java | 8 ++++++-- .../peertube/PeertubeStreamExtractorDefaultTest.java | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index 4dadae7785..a57720e344 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -76,7 +76,7 @@ public String getDescription() throws ParsingException { try { desc = JsonUtils.getString(json, "description"); } catch (ParsingException e) { - return "No description"; + return ""; } if (desc.length() == 250 && desc.substring(247).equals("...")) { //if description is shortened, get full description @@ -256,7 +256,11 @@ public List getTags(){ @Nonnull @Override public String getSupportInfo() throws ParsingException { - return JsonUtils.getString(json, "support"); + try { + return JsonUtils.getString(json, "support"); + } catch (ParsingException e) { + return ""; + } } private String getRelatedStreamsUrl(List tags) throws UnsupportedEncodingException { diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java index 8296785262..962735062f 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java @@ -61,7 +61,7 @@ public void testGetLargeDescription() throws ParsingException { public void testGetEmptyDescription() throws Exception { PeertubeStreamExtractor extractorEmpty = (PeertubeStreamExtractor) PeerTube.getStreamExtractor("https://framatube.org/api/v1/videos/d5907aad-2252-4207-89ec-a4b687b9337d"); extractorEmpty.fetchPage(); - assertEquals("No description", extractorEmpty.getDescription()); + assertEquals("", extractorEmpty.getDescription()); } @Test From a691d6d0fc4949ab1392082865416e83a04af341 Mon Sep 17 00:00:00 2001 From: bopol Date: Thu, 23 Jan 2020 19:02:40 +0100 Subject: [PATCH 08/18] fix upload date: there was a one hour delay on peertube.co.uk: https://i.imgur.com/8Pai1rb.png on newpipe, before this commit: https://i.imgur.com/NIRbs4Z.png on newpipe, with this commit: https://i.ibb.co/mhKKtRD/Screenshot-20200123-185422-New-Pipe-Debug.png --- .../extractor/services/peertube/PeertubeParsingHelper.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java index f518faf9ae..2483dd9495 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java @@ -4,6 +4,7 @@ import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.Date; +import java.util.concurrent.TimeUnit; import org.jsoup.helper.StringUtil; import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; @@ -27,6 +28,7 @@ public static Calendar parseDateFrom(String textualUploadDate) throws ParsingExc Date date; try { date = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'").parse(textualUploadDate); + date = new Date(date.getTime() + TimeUnit.HOURS.toMillis(1)); } catch (ParseException e) { throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"", e); } From c790261eed4ba148b603325075bcbbfbd3e9b4c9 Mon Sep 17 00:00:00 2001 From: bopol Date: Thu, 23 Jan 2020 19:08:41 +0100 Subject: [PATCH 09/18] update test --- .../services/peertube/PeertubeStreamExtractorDefaultTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java index 962735062f..c6a164f742 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorDefaultTest.java @@ -91,7 +91,7 @@ public void testGetViewCount() throws ParsingException { @Test public void testGetUploadDate() throws ParsingException, ParseException { final Calendar instance = Calendar.getInstance(); - instance.setTime(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'").parse("2018-10-01T10:52:46.396Z")); + instance.setTime(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'").parse("2018-10-01T11:52:46.396Z")); assertEquals(instance, requireNonNull(extractor.getUploadDate()).date()); } From 812c8e0ad25de2f1d3f5da16975cae1b145072f1 Mon Sep 17 00:00:00 2001 From: B0pol Date: Fri, 24 Jan 2020 00:22:05 +0100 Subject: [PATCH 10/18] authorName in comments now follow PeerTube website MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit https://peertube.cpy.re/videos/watch/d2a5ec78-5f85-4090-8ec5-dc1102e022ea anonyme zirbeldrüse is his displayName, what was displayed in NewPipe. but on the website, it's shown 777@mastodon.xyz --- .../extractors/PeertubeCommentsInfoItemExtractor.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java index edb59e1fe0..0bcce9fc47 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java @@ -67,7 +67,7 @@ public String getCommentText() throws ParsingException { try { Document doc = Jsoup.parse(htmlText); return doc.body().text(); - }catch(Exception e) { + } catch(Exception e) { return htmlText.replaceAll("(?s)<[^>]*>(\\s*<[^>]*>)*", ""); } } @@ -83,7 +83,7 @@ public String getAuthorThumbnail() throws ParsingException { String value; try { value = JsonUtils.getString(item, "account.avatar.path"); - }catch(Exception e) { + } catch(Exception e) { value = "/client/assets/images/default-avatar.png"; } return baseUrl + value; @@ -91,7 +91,7 @@ public String getAuthorThumbnail() throws ParsingException { @Override public String getAuthorName() throws ParsingException { - return JsonUtils.getString(item, "account.displayName"); + return JsonUtils.getString(item, "account.name") + "@" + JsonUtils.getString(item, "account.host"); } @Override From 341372c0d0dadcce1b88333d9c8afc58ec50fff7 Mon Sep 17 00:00:00 2001 From: B0pol Date: Fri, 24 Jan 2020 20:16:24 +0100 Subject: [PATCH 11/18] reindenting (ctrl alt l) on JsonUtils and PeertubeStreamExtractor --- .../extractors/PeertubeStreamExtractor.java | 75 ++++++++++--------- .../newpipe/extractor/utils/JsonUtils.java | 46 ++++++------ 2 files changed, 61 insertions(+), 60 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index a57720e344..e6e32d44a8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -39,16 +39,16 @@ public class PeertubeStreamExtractor extends StreamExtractor { - + + private final String baseUrl; private JsonObject json; private List subtitles = new ArrayList<>(); - private final String baseUrl; - + public PeertubeStreamExtractor(StreamingService service, LinkHandler linkHandler) throws ParsingException { super(service, linkHandler); this.baseUrl = getBaseUrl(); } - + @Override public String getTextualUploadDate() throws ParsingException { return JsonUtils.getString(json, "publishedAt"); @@ -64,7 +64,7 @@ public DateWrapper getUploadDate() throws ParsingException { return new DateWrapper(PeertubeParsingHelper.parseDateFrom(textualUploadDate)); } - + @Override public String getThumbnailUrl() throws ParsingException { return baseUrl + JsonUtils.getString(json, "previewPath"); @@ -149,7 +149,7 @@ public String getUploaderAvatarUrl() throws ParsingException { String value; try { value = JsonUtils.getString(json, "account.avatar.path"); - }catch(Exception e) { + } catch (Exception e) { value = "/client/assets/images/default-avatar.png"; } return baseUrl + value; @@ -176,8 +176,8 @@ public List getVideoStreams() throws IOException, ExtractionExcepti List videoStreams = new ArrayList<>(); try { JsonArray streams = json.getArray("files", new JsonArray()); - for(Object s: streams) { - if(!(s instanceof JsonObject)) continue; + for (Object s : streams) { + if (!(s instanceof JsonObject)) continue; JsonObject stream = (JsonObject) s; String url = JsonUtils.getString(stream, "fileUrl"); String torrentUrl = JsonUtils.getString(stream, "torrentUrl"); @@ -211,8 +211,8 @@ public List getSubtitlesDefault() throws IOException, Extractio @Override public List getSubtitles(final MediaFormat format) throws IOException, ExtractionException { List filteredSubs = new ArrayList<>(); - for(SubtitlesStream sub: subtitles) { - if(sub.getFormat() == format) { + for (SubtitlesStream sub : subtitles) { + if (sub.getFormat() == format) { filteredSubs.add(sub); } } @@ -234,18 +234,18 @@ public StreamInfoItemsCollector getRelatedStreams() throws IOException, Extracti StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); List tags = getTags(); String apiUrl = null; - if(!tags.isEmpty()) { + if (!tags.isEmpty()) { apiUrl = getRelatedStreamsUrl(tags); - - }else { + + } else { apiUrl = getUploaderUrl() + "/videos?start=0&count=8"; } - if(!StringUtil.isBlank(apiUrl)) getStreamsFromApi(collector, apiUrl); + if (!StringUtil.isBlank(apiUrl)) getStreamsFromApi(collector, apiUrl); return collector; } @Override - public List getTags(){ + public List getTags() { try { return (List) JsonUtils.getArray(json, "tags"); } catch (Exception e) { @@ -267,7 +267,7 @@ private String getRelatedStreamsUrl(List tags) throws UnsupportedEncodin String url = baseUrl + PeertubeSearchQueryHandlerFactory.SEARCH_ENDPOINT; StringBuilder params = new StringBuilder(); params.append("start=0&count=8&sort=-createdAt"); - for(String tag : tags) { + for (String tag : tags) { params.append("&tagsOneOf="); params.append(URLEncoder.encode(tag, "UTF-8")); } @@ -277,38 +277,38 @@ private String getRelatedStreamsUrl(List tags) throws UnsupportedEncodin private void getStreamsFromApi(StreamInfoItemsCollector collector, String apiUrl) throws ReCaptchaException, IOException, ParsingException { Response response = getDownloader().get(apiUrl); JsonObject relatedVideosJson = null; - if(null != response && !StringUtil.isBlank(response.responseBody())) { + if (null != response && !StringUtil.isBlank(response.responseBody())) { try { relatedVideosJson = JsonParser.object().from(response.responseBody()); } catch (JsonParserException e) { throw new ParsingException("Could not parse json data for related videos", e); } } - - if(relatedVideosJson != null) { + + if (relatedVideosJson != null) { collectStreamsFrom(collector, relatedVideosJson); } } - + private void collectStreamsFrom(StreamInfoItemsCollector collector, JsonObject json) throws ParsingException { JsonArray contents; try { contents = (JsonArray) JsonUtils.getValue(json, "data"); - }catch(Exception e) { + } catch (Exception e) { throw new ParsingException("unable to extract related videos", e); } - - for(Object c: contents) { - if(c instanceof JsonObject) { + + for (Object c : contents) { + if (c instanceof JsonObject) { final JsonObject item = (JsonObject) c; PeertubeStreamInfoItemExtractor extractor = new PeertubeStreamInfoItemExtractor(item, baseUrl); //do not add the same stream in related streams - if(!extractor.getUrl().equals(getUrl())) collector.commit(extractor); + if (!extractor.getUrl().equals(getUrl())) collector.commit(extractor); } } - + } - + @Override public String getErrorMessage() { @@ -318,12 +318,12 @@ public String getErrorMessage() { @Override public void onFetchPage(Downloader downloader) throws IOException, ExtractionException { Response response = downloader.get(getUrl()); - if(null != response && null != response.responseBody()) { + if (null != response && null != response.responseBody()) { setInitialData(response.responseBody()); - }else { + } else { throw new ExtractionException("Unable to extract peertube channel data"); } - + loadSubtitles(); } @@ -333,24 +333,25 @@ private void setInitialData(String responseBody) throws ExtractionException { } catch (JsonParserException e) { throw new ExtractionException("Unable to extract peertube stream data", e); } - if(null == json) throw new ExtractionException("Unable to extract peertube stream data"); + if (null == json) throw new ExtractionException("Unable to extract peertube stream data"); PeertubeParsingHelper.validate(json); } - + private void loadSubtitles() { if (subtitles.isEmpty()) { try { - Response response = getDownloader().get(getUrl() + "/captions"); + Response response = getDownloader().get(getUrl() + "/captions"); JsonObject captionsJson = JsonParser.object().from(response.responseBody()); JsonArray captions = JsonUtils.getArray(captionsJson, "data"); - for(Object c: captions) { - if(c instanceof JsonObject) { - JsonObject caption = (JsonObject)c; + for (Object c : captions) { + if (c instanceof JsonObject) { + JsonObject caption = (JsonObject) c; String url = baseUrl + JsonUtils.getString(caption, "captionPath"); String languageCode = JsonUtils.getString(caption, "language.id"); String ext = url.substring(url.lastIndexOf(".") + 1); MediaFormat fmt = MediaFormat.getFromSuffix(ext); - if(fmt != null && languageCode != null) subtitles.add(new SubtitlesStream(fmt, languageCode, url, false)); + if (fmt != null && languageCode != null) + subtitles.add(new SubtitlesStream(fmt, languageCode, url, false)); } } } catch (Exception e) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java index ebd251d7ce..80e0dd58f5 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java @@ -16,71 +16,71 @@ public class JsonUtils { private JsonUtils() { } - + @Nonnull - public static Object getValue(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{ + public static Object getValue(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException { List keys = Arrays.asList(path.split("\\.")); object = getObject(object, keys.subList(0, keys.size() - 1)); if (null == object) throw new ParsingException("Unable to get " + path); Object result = object.get(keys.get(keys.size() - 1)); - if(null == result) throw new ParsingException("Unable to get " + path); + if (null == result) throw new ParsingException("Unable to get " + path); return result; } - + @Nonnull - public static String getString(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{ + public static String getString(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException { Object value = getValue(object, path); - if(value instanceof String) { + if (value instanceof String) { return (String) value; - }else { + } else { throw new ParsingException("Unable to get " + path); } } @Nonnull - public static Boolean getBoolean(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{ + public static Boolean getBoolean(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException { Object value = getValue(object, path); - if(value instanceof Boolean) { + if (value instanceof Boolean) { return (Boolean) value; - }else { + } else { throw new ParsingException("Unable to get " + path); } } - + @Nonnull - public static Number getNumber(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{ + public static Number getNumber(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException { Object value = getValue(object, path); - if(value instanceof Number) { + if (value instanceof Number) { return (Number) value; - }else { + } else { throw new ParsingException("Unable to get " + path); } } - + @Nonnull - public static JsonObject getObject(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{ + public static JsonObject getObject(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException { Object value = getValue(object, path); - if(value instanceof JsonObject) { + if (value instanceof JsonObject) { return (JsonObject) value; - }else { + } else { throw new ParsingException("Unable to get " + path); } } - + @Nonnull - public static JsonArray getArray(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException{ + public static JsonArray getArray(@Nonnull JsonObject object, @Nonnull String path) throws ParsingException { Object value = getValue(object, path); - if(value instanceof JsonArray) { + if (value instanceof JsonArray) { return (JsonArray) value; - }else { + } else { throw new ParsingException("Unable to get " + path); } } @Nonnull public static List