[reactor] detect "circular" redirects (#148)

gallery_dl/extractor/reactor.py

@@ -63,7 +63,18 @@ def posts(self):
     def _pagination(self, url):
         while True:
             time.sleep(random.uniform(self.wait_min, self.wait_max))
-            page = self.request(url).text
+
+            response = self.request(url)
+            if response.history:
+                # sometimes there is a redirect from
+                # the last page of a listing (.../tag/<tag>/1)
+                # to the first page (.../tag/<tag>)
+                # which could cause an endless loop
+                cnt_old = response.history[0].url.count("/")
+                cnt_new = response.url.count("/")
+                if cnt_old == 5 and cnt_new == 4:
+                    return
+            page = response.text
 
             yield from text.extract_iter(
                 page, '<div class="uhead">', '<div class="ufoot">')

test/test_results.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
-# Copyright 2015-2018 Mike Fährmann
+# Copyright 2015-2019 Mike Fährmann
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as
@@ -16,7 +16,7 @@
 
 # these don't work on Travis CI
 TRAVIS_SKIP = {
-    "exhentai", "kissmanga", "mangafox", "dynastyscans", "nijie",
+    "exhentai", "kissmanga", "mangafox", "dynastyscans", "nijie", "bobx",
     "archivedmoe", "archiveofsins", "thebarchive", "fireden",
     "sankaku", "idolcomplex", "mangahere", "readcomiconline",
 }