[PeerTube] added metadata, fix descriptions, fix thumbnail, fix upload date, fix age limit, update tests.

README.md

@@ -21,7 +21,7 @@ To test changes quickly you can build the library locally. Using the local Maven
 2. It's _recommended_ that you change the `version` of this library (e.g. `LOCAL_SNAPSHOT`).
 3. Run gradle's `ìnstall` task to deploy this library to your local repository (using the wrapper, present in the root of this project: `./gradlew install`)
 4. Change the dependency version used in your project to match the one you chose in step 2 (`implementation 'com.github.TeamNewPipe:NewPipeExtractor:LOCAL_SNAPSHOT'`)
-     
+
 > Tip for Android Studio users: After you make changes and run the `install` task, use the menu option `File → "Sync with File System"` to refresh the library in your project.
 
 ## Supported sites

extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java

@@ -12,6 +12,7 @@
 import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
 import org.schabi.newpipe.extractor.localization.DateWrapper;
 import org.schabi.newpipe.extractor.stream.*;
+import org.schabi.newpipe.extractor.utils.JsonUtils;
 
 import javax.annotation.Nonnull;
 import java.io.IOException;
@@ -225,4 +226,41 @@ public String getName() throws ParsingException {
     public String getOriginalUrl() throws ParsingException {
         return data.getString("frontend_link");
     }
+
+    @Override
+    public String getHost() throws ParsingException {
+        return "";
+    }
+
+    @Override
+    public String getPrivacy() throws ParsingException {
+        return "";
+    }
+
+    @Override
+    public String getCategory() throws ParsingException {
+        return "";
+    }
+
+    @Override
+    public String getLicence() throws ParsingException {
+        return "";
+    }
+
+    @Override
+    public String getLanguageInfo() throws ParsingException {
+        return "";
+    }
+
+    @Nonnull
+    @Override
+    public List<String> getTags() throws ParsingException {
+        return new ArrayList<>();
+    }
+
+    @Nonnull
+    @Override
+    public String getSupportInfo() throws ParsingException {
+        return "";
+    }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java

@@ -4,6 +4,7 @@
 import java.text.SimpleDateFormat;
 import java.util.Calendar;
 import java.util.Date;
+import java.util.concurrent.TimeUnit;
 
 import org.jsoup.helper.StringUtil;
 import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
@@ -27,6 +28,7 @@ public static Calendar parseDateFrom(String textualUploadDate) throws ParsingExc
         Date date;
         try {
             date = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'").parse(textualUploadDate);
+            date = new Date(date.getTime() + TimeUnit.HOURS.toMillis(1));
         } catch (ParseException e) {
             throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"", e);
         }

extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java

@@ -67,7 +67,7 @@ public String getCommentText() throws ParsingException {
         try {
             Document doc = Jsoup.parse(htmlText);
             return doc.body().text();
-        }catch(Exception e) {
+        } catch(Exception e) {
             return htmlText.replaceAll("(?s)<[^>]*>(\\s*<[^>]*>)*", "");
         }
     }
@@ -83,15 +83,15 @@ public String getAuthorThumbnail() throws ParsingException {
         String value;
         try {
             value = JsonUtils.getString(item, "account.avatar.path");
-        }catch(Exception e) {
+        } catch(Exception e) {
             value = "/client/assets/images/default-avatar.png";
         }
         return baseUrl + value;
     }
 
     @Override
     public String getAuthorName() throws ParsingException {
-        return JsonUtils.getString(item, "account.displayName");
+        return JsonUtils.getString(item, "account.name") + "@" + JsonUtils.getString(item, "account.host");
     }
 
     @Override

extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java

@@ -9,7 +9,7 @@
 
 import org.jsoup.helper.StringUtil;
 import org.schabi.newpipe.extractor.MediaFormat;
-import org.schabi.newpipe.extractor.ServiceList;
+import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.downloader.Downloader;
 import org.schabi.newpipe.extractor.downloader.Response;
@@ -29,13 +29,14 @@
 import org.schabi.newpipe.extractor.stream.SubtitlesStream;
 import org.schabi.newpipe.extractor.stream.VideoStream;
 import org.schabi.newpipe.extractor.utils.JsonUtils;
-import org.schabi.newpipe.extractor.utils.Utils;
 
 import com.grack.nanojson.JsonArray;
 import com.grack.nanojson.JsonObject;
 import com.grack.nanojson.JsonParser;
 import com.grack.nanojson.JsonParserException;
 
+import javax.annotation.Nonnull;
+
 public class PeertubeStreamExtractor extends StreamExtractor {
 
 
@@ -66,21 +67,39 @@ public DateWrapper getUploadDate() throws ParsingException {
 
     @Override
     public String getThumbnailUrl() throws ParsingException {
-        return baseUrl + JsonUtils.getString(json, "thumbnailPath");
+        return baseUrl + JsonUtils.getString(json, "previewPath");
     }
 
     @Override
     public String getDescription() throws ParsingException {
+        String desc;
         try {
-            return JsonUtils.getString(json, "description");
-        }catch(ParsingException e) {
-            return "No description";
+            desc = JsonUtils.getString(json, "description");
+        } catch (ParsingException e) {
+            return "";
+        }
+        if (desc.length() == 250 && desc.substring(247).equals("...")) {
+            //if description is shortened, get full description
+            Downloader dl = NewPipe.getDownloader();
+            try {
+                Response response = dl.get(getUrl() + "/description");
+                JsonObject jsonObject = JsonParser.object().from(response.responseBody());
+                desc = JsonUtils.getString(jsonObject, "description");
+            } catch (ReCaptchaException | IOException | JsonParserException e) {
+                e.printStackTrace();
+            }
         }
+        return desc;
     }
 
     @Override
     public int getAgeLimit() throws ParsingException {
-        return NO_AGE_LIMIT;
+        boolean isNSFW = JsonUtils.getBoolean(json, "nsfw");
+        if (isNSFW) {
+            return 18;
+        } else {
+            return NO_AGE_LIMIT;
+        }
     }
 
     @Override
@@ -224,15 +243,26 @@ public StreamInfoItemsCollector getRelatedStreams() throws IOException, Extracti
         if(!StringUtil.isBlank(apiUrl)) getStreamsFromApi(collector, apiUrl);
         return collector;
     }
-
-    private List<String> getTags(){
+
+    @Override
+    public List<String> getTags(){
         try {
             return (List) JsonUtils.getArray(json, "tags");
         } catch (Exception e) {
             return Collections.emptyList();
         }
     }
-
+
+    @Nonnull
+    @Override
+    public String getSupportInfo() throws ParsingException {
+        try {
+            return JsonUtils.getString(json, "support");
+        } catch (ParsingException e) {
+            return "";
+        }
+    }
+
     private String getRelatedStreamsUrl(List<String> tags) throws UnsupportedEncodingException {
         String url = baseUrl + PeertubeSearchQueryHandlerFactory.SEARCH_ENDPOINT;
         StringBuilder params = new StringBuilder();
@@ -339,4 +369,28 @@ public String getOriginalUrl() throws ParsingException {
         return baseUrl + "/videos/watch/" + getId();
     }
 
+    @Override
+    public String getHost() throws ParsingException {
+        return JsonUtils.getString(json, "account.host");
+    }
+
+    @Override
+    public String getPrivacy() throws ParsingException {
+        return JsonUtils.getString(json, "privacy.label");
+    }
+
+    @Override
+    public String getCategory() throws ParsingException {
+        return JsonUtils.getString(json, "category.label");
+    }
+
+    @Override
+    public String getLicence() throws ParsingException {
+        return JsonUtils.getString(json, "licence.label");
+    }
+
+    @Override
+    public String getLanguageInfo() throws ParsingException {
+        return JsonUtils.getString(json, "language.label");
+    }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java

@@ -254,4 +254,41 @@ public StreamInfoItemsCollector getRelatedStreams() throws IOException, Extracti
     public String getErrorMessage() {
         return null;
     }
+
+    @Override
+    public String getHost() throws ParsingException {
+        return "";
+    }
+
+    @Override
+    public String getPrivacy() throws ParsingException {
+        return "";
+    }
+
+    @Override
+    public String getCategory() throws ParsingException {
+        return "";
+    }
+
+    @Override
+    public String getLicence() throws ParsingException {
+        return "";
+    }
+
+    @Override
+    public String getLanguageInfo() throws ParsingException {
+        return "";
+    }
+
+    @Nonnull
+    @Override
+    public List<String> getTags() throws ParsingException {
+        return new ArrayList<>();
+    }
+
+    @Nonnull
+    @Override
+    public String getSupportInfo() throws ParsingException {
+        return "";
+    }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java

@@ -1134,4 +1134,41 @@ public List<Frameset> getFrames() throws ExtractionException {
 			throw new ExtractionException(e);
 		}
 	}
+
+    @Override
+    public String getHost() throws ParsingException {
+        return "";
+    }
+
+    @Override
+    public String getPrivacy() throws ParsingException {
+        return "";
+    }
+
+    @Override
+    public String getCategory() throws ParsingException {
+        return "";
+    }
+
+    @Override
+    public String getLicence() throws ParsingException {
+        return "";
+    }
+
+    @Override
+    public String getLanguageInfo() throws ParsingException {
+        return "";
+    }
+
+    @Nonnull
+    @Override
+    public List<String> getTags() throws ParsingException {
+        return new ArrayList<>();
+    }
+
+    @Nonnull
+    @Override
+    public String getSupportInfo() throws ParsingException {
+        return "";
+    }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java

@@ -27,6 +27,7 @@
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
 import org.schabi.newpipe.extractor.localization.DateWrapper;
+import org.schabi.newpipe.extractor.utils.JsonUtils;
 import org.schabi.newpipe.extractor.utils.Parser;
 
 import javax.annotation.Nonnull;
@@ -349,4 +350,72 @@ protected long getTimestampSeconds(String regexPattern) throws ParsingException
             return 0;
         }
     }
+
+    /**
+     * The host of the stream (Eg. peertube.cpy.re).
+     * If the host is not available, or if the service doesn't use
+     * a federated system, but a centralised system,
+     * you can simply return an empty string.
+     * @return the host of the stream or an empty String.
+     * @throws ParsingException
+     */
+    @Nonnull
+    public abstract String getHost() throws ParsingException;
+
+    /**
+     * The privacy of the stream (Eg. Public, Private, Unlisted…).
+     * If the privacy is not available you can simply return an empty string.
+     * @return the privacy of the stream or an empty String.
+     * @throws ParsingException
+     */
+    @Nonnull
+    public abstract String getPrivacy() throws ParsingException;
+
+    /**
+     * The name of the category of the stream.
+     * If the category is not available you can simply return an empty string.
+     * @return the category of the stream or an empty String.
+     * @throws ParsingException
+     */
+    @Nonnull
+    public abstract String getCategory() throws ParsingException;
+
+    /**
+     * The name of the licence of the stream.
+     * If the licence is not available you can simply return an empty string.
+     * @return the licence of the stream or an empty String.
+     * @throws ParsingException
+     */
+    @Nonnull
+    public abstract String getLicence() throws ParsingException;
+
+    /**
+     * The language of the stream.
+     * If the language is not available you can simply return an empty string.
+     * @return the licence of the stream or an empty String.
+     * @throws ParsingException
+     */
+    @Nonnull
+    public abstract String getLanguageInfo() throws ParsingException;
+
+    /**
+     * The list of tags of the stream.
+     * If the tag list is not available you can simply return an empty list.
+     * @return the list of tags of the stream or an empty list.
+     * @throws ParsingException
+     */
+    @Nonnull
+    public abstract List<String> getTags() throws ParsingException;
+
+    /**
+     * The support information of the stream.
+     * see: https://framatube.org/videos/watch/ee408ec8-07cd-4e35-b884-fb681a4b9d37
+     * (support button).
+     * If the support information are not available,
+     * you can simply return an empty String.
+     * @return the support information of the stream or an empty String.
+     * @throws ParsingException
+     */
+    @Nonnull
+    public abstract String getSupportInfo() throws ParsingException;
 }