[twitter] add 'list-members' extractor (closes #1096)

docs/supportedsites.rst

@@ -162,5 +162,5 @@ Turboimagehost       https://www.turboimagehost.com/     individual Images
 .. |pixiv-C| replace:: Favorites, Follows, pixiv.me Links, Rankings, Search Results, User Profiles, individual Images
 .. |reddit-C| replace:: individual Images, Submissions, Subreddits, User Profiles
 .. |smugmug-C| replace:: Albums, individual Images, Images from Users and Folders
-.. |twitter-C| replace:: Bookmarks, Likes, Media Timelines, Search Results, Timelines, Tweets
+.. |twitter-C| replace:: Bookmarks, Likes, Lists, List Members, Media Timelines, Search Results, Timelines, Tweets
 .. |yuki-S| replace:: yuki.la 4chan archive

gallery_dl/extractor/twitter.py

@@ -11,7 +11,7 @@
 from .common import Extractor, Message
 from .. import text, util, exception
 from ..cache import cache
-
+import json
 
 BASE_PATTERN = (
     r"(?:https?://)?(?:www\.|mobile\.)?"
@@ -324,7 +324,7 @@ def tweets(self):
 class TwitterListExtractor(TwitterExtractor):
     """Extractor for Twitter lists"""
     subcategory = "list"
-    pattern = BASE_PATTERN + r"/i/lists/(\d+)"
+    pattern = BASE_PATTERN + r"/i/lists/(\d+)/?$"
     test = ("https://twitter.com/i/lists/784214683683127296", {
         "range": "1-40",
         "count": 40,
@@ -335,6 +335,21 @@ def tweets(self):
         return TwitterAPI(self).timeline_list(self.user)
 
 
+class TwitterListMembersExtractor(TwitterExtractor):
+    """Extractor for members of a Twitter list"""
+    subcategory = "list-members"
+    pattern = BASE_PATTERN + r"/i/lists/(\d+)/members"
+    test = ("https://twitter.com/i/lists/784214683683127296/members",)
+
+    def items(self):
+        self.login()
+        for user in TwitterAPI(self).list_members(self.user):
+            user["_extractor"] = TwitterTimelineExtractor
+            url = "{}/intent/user?user_id={}".format(
+                self.root, user["rest_id"])
+            yield Message.Queue, url, user
+
+
 class TwitterSearchExtractor(TwitterExtractor):
     """Extractor for all images from a search timeline"""
     subcategory = "search"
@@ -543,6 +558,16 @@ def search(self, query):
         return self._pagination(
             endpoint, params, "sq-I-t-", "sq-cursor-bottom")
 
+    def list_members(self, list_id):
+        endpoint = "graphql/M74V2EwlxxVYGB4DbyAphQ/ListMembers"
+        variables = {
+            "listId": list_id,
+            "count" : 20,
+            "withTweetResult": False,
+            "withUserResult" : False,
+        }
+        return self._pagination_members(endpoint, variables)
+
     def list_by_rest_id(self, list_id):
         endpoint = "graphql/LXXTUytSX1QY-2p8Xp9BFA/ListByRestId"
         params = {"variables": '{"listId":"' + list_id + '"'
@@ -655,3 +680,30 @@ def _pagination(self, endpoint, params=None,
             if not cursor or not tweet:
                 return
             params["cursor"] = cursor
+
+    def _pagination_members(self, endpoint, variables):
+        while True:
+            cursor = entry = stop = None
+            params = {"variables": json.dumps(variables)}
+            data = self._call(endpoint, params)
+
+            try:
+                instructions = (data["data"]["list"]["members_timeline"]
+                                ["timeline"]["instructions"])
+            except KeyError:
+                raise exception.AuthorizationError()
+
+            for instr in instructions:
+                if instr["type"] == "TimelineAddEntries":
+                    for entry in instr["entries"]:
+                        if entry["entryId"].startswith("user-"):
+                            yield entry["content"]["itemContent"]["user"]
+                        elif entry["entryId"].startswith("cursor-bottom-"):
+                            cursor = entry["content"]["value"]
+                elif instr["type"] == "TimelineTerminateTimeline":
+                    if instr["direction"] == "Bottom":
+                        stop = True
+
+            if stop or not cursor or not entry:
+                return
+            variables["cursor"] = cursor

scripts/supportedsites.py

@@ -142,6 +142,7 @@
     },
     "twitter": {
         "media": "Media Timelines",
+        "list-members": "List Members",
     },
     "wikiart": {
         "artists": "Artist Listings",