[twitter] ignore previously seen Tweets (#2712)

occurs primarily for /with_replies results when logged in
mikf · Jul 3, 2022 · 1d14928 · 1d14928
1 parent 4b2a0a0
commit 1d14928
Show file tree

Hide file tree

Showing 3 changed files with 21 additions and 0 deletions.
diff --git a/docs/configuration.rst b/docs/configuration.rst
@@ -2415,6 +2415,16 @@ Description
     Extract `TwitPic <https://twitpic.com/>`__ embeds.
 
 
+extractor.twitter.unique
+------------------------
+Type
+    ``bool``
+Default
+    ``true``
+Description
+    Ignore previously seen Tweets.
+
+
 extractor.twitter.users
 -----------------------
 Type

diff --git a/docs/gallery-dl.conf b/docs/gallery-dl.conf
@@ -288,6 +288,7 @@
             "strategy": null,
             "text-tweets": false,
             "twitpic": false,
+            "unique": true,
             "users": "timeline",
             "videos": true
         },

diff --git a/gallery_dl/extractor/twitter.py b/gallery_dl/extractor/twitter.py
@@ -64,13 +64,23 @@ def items(self):
             tweets = self._expand_tweets(self.tweets())
             self.tweets = lambda : tweets
 
+        if self.config("unique", True):
+            seen_tweets = set()
+        else:
+            seen_tweets = None
+
         for tweet in self.tweets():
 
             if "legacy" in tweet:
                 data = tweet["legacy"]
             else:
                 data = tweet
 
+            if seen_tweets is not None:
+                if data["id_str"] in seen_tweets:
+                    continue
+                seen_tweets.add(data["id_str"])
+
             if not self.retweets and "retweeted_status_id_str" in data:
                 self.log.debug("Skipping %s (retweet)", data["id_str"])
                 continue