Fix SplashRequest

Author: kmike

scrapyjs/middleware.py

@@ -36,8 +36,10 @@ def __init__(self, crawler, splash_base_url, slot_policy):
 
     @classmethod
     def from_crawler(cls, crawler):
-        splash_base_url = crawler.settings.get('SPLASH_URL', cls.default_splash_url)
-        slot_policy = crawler.settings.get('SPLASH_SLOT_POLICY', cls.default_policy)
+        splash_base_url = crawler.settings.get('SPLASH_URL',
+                                               cls.default_splash_url)
+        slot_policy = crawler.settings.get('SPLASH_SLOT_POLICY',
+                                           cls.default_policy)
 
         if slot_policy not in SlotPolicy._known:
             raise NotConfigured("Incorrect slot policy: %r" % slot_policy)
@@ -49,17 +51,20 @@ def process_request(self, request, spider):
         if not splash_options:
             return
 
+        if request.meta.get("_splash_processed"):
+            # don't process the same request more than once
+            return
+
         if request.method != 'GET':
             logger.warn(
-                "Currently only GET requests are supported by SplashMiddleware; "
-                "%(request)s will be handled without Splash",
+                "Currently only GET requests are supported by SplashMiddleware;"
+                " %(request)s will be handled without Splash",
                 {'request': request},
                 extra={'spider': spider}
             )
             return request
 
         meta = request.meta
-        del meta['splash']
         meta['_splash_processed'] = splash_options
 
         slot_policy = splash_options.get('slot_policy', self.slot_policy)
@@ -83,7 +88,9 @@ def process_request(self, request, spider):
             # But we can change Scrapy `download_timeout`: increase
             # it when it's too small. Decreasing `download_timeout` is not
             # safe.
-            timeout_current = meta.get('download_timeout', 1e6)  # no timeout means infinite timeout
+
+            # no timeout means infinite timeout
+            timeout_current = meta.get('download_timeout', 1e6)
             timeout_expected = float(args['timeout']) + self.splash_extra_timeout
 
             if timeout_expected > timeout_current:
@@ -131,4 +138,6 @@ def _set_download_slot(self, request, meta, slot_policy):
             pass
 
     def _get_slot_key(self, request_or_response):
-        return self.crawler.engine.downloader._get_slot_key(request_or_response, None)
+        return self.crawler.engine.downloader._get_slot_key(
+            request_or_response, None
+        )

scrapyjs/request.py

@@ -23,6 +23,11 @@ def __init__(self, url=None, *args, **kwargs):
             meta['splash'] = copy.deepcopy(self.default_splash_meta)
         super(SplashRequest, self).__init__(url, *args, meta=meta, **kwargs)
 
+    def replace(self, *args, **kwargs):
+        cls = super(SplashRequest, self).replace(*args, **kwargs)
+        cls._original_url = self._original_url
+        return cls
+
     def __str__(self):
         return "<%s %s %s>" % (self.method, self.url, self._original_url)