scrapy.Reaquests（）回调不起作用

Question

虽然有很多问题，但由于“dont_filter”参数，大多数人遇到此问题，我通过了这个参数“dont_filter = True”，但我的自定义解析生成器仍然没有工作，下面是我的代码（第三个解析器“parse_spec”从来没有被调用，“parse_models_follow_next_page”在被parse（）调用时工作得很好，但当它需要转到下一页时它不能调用自己） ：

import scrapy 
from gsmarena.items import PhoneItems 

class VendorSpider(scrapy.Spider): 
    custom_settings = { 
     'DOWNLOAD_DELAY': 1.5, 
     'USER_AGENT': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/7046A194A', 
     'COOKIES_ENABLED': False 
    } 

    name = "gsmarena_spec" 

    allowed_domains = ["https://www.gsmarena.com/"] 

    start_urls = [ 
     "https://www.gsmarena.com/makers.php3" 
    ] 

    def parse(self, response): 
     # print("Existing settings: %s" % self.settings.attributes.items()) 
     length = len(response.xpath("//table//a").extract()) 
     for i in range(1, length): 
      brand = response.xpath(
       '(//table//a)[{}]/text()'.format(i)).extract()[0] 
      url = "https://www.gsmarena.com/" + \ 
       response.xpath("(//table//a)[{}]/@href".format(i)).extract()[0] 
      yield scrapy.Request(url, callback=self.parse_models_follow_next_page, meta={'brand': brand}, dont_filter=True) 

    def parse_models_follow_next_page(self, response): 
     brand = response.meta.get('brand') 
     length = len(response.xpath(
      "//div[@class='makers']/self::div//a").extract()) 
     for i in range(1, length): 
      url = "https://www.gsmarena.com/" + \ 
       response.xpath(
        "(//div[@class='makers']/self::div//a)[{}]/@href".format(i)).extract()[0] 
      model = response.xpath(
       "(//div[@class='makers']/self::div//a//span/text())[{}]".format(i)).extract()[0] 
      yield scrapy.Request(url, callback=self.parse_spec, meta={'brand': brand, 'model': model}, dont_filter=True) 
     is_next_page = response.xpath(
      "//a[@class=\"pages-next\"]/@href").extract() 

     if is_next_page: 
      next_page = "https://www.gsmarena.com/" + is_next_page[0] 
      yield scrapy.Request(next_page, callback=self.parse_models_follow_next_page, meta={'brand': brand}, dont_filter=True) 



    def parse_spec(self, response): 
     item = PhoneItems() 
     item['model'] = response.meta.get('model') 
     item['brand'] = response.meta.get('brand') 
     for spec_name, spec in zip(response.xpath('//table//td[1]').extract(), response.xpath('//table//td[2]').extract()): 
     item[spec_name] = spec 
     yield item 

和我的英语不好对不起

Answer 1

你刷屏有几个问题。

allowed_domains = ["https://www.gsmarena.com/"] 

应该

allowed_domains = ["www.gsmarena.com"] 

下一页你没有在你的类中定义

def errback_httpbin(self, response): 
    pass 

下面的代码errback_httpbin方法

for spec_name, spec in zip(response.xpath('//table//td[1]').extract(), response.xpath('//table//td[2]').extract()): 

应该是

for spec_name, spec in zip(response.xpath('//table//td[1]/text()').extract(), response.xpath('//table//td[2]/text()').extract()): 

这虽然仍然有一些问题。

而且你的代码会需要一些时间，第一产量，用作调度将基于URL的在未来的顺序挑选的网址

Answer 2

我已经在代码中的一些变化，它废料全部结果预计spec_name，即没有以理解的方式指定。

进口scrapy

从LXML导入HTML

从tutorial.items进口PhoneItems

类VendorSpider（scrapy.Spider）：

custom_settings = { 
    'DOWNLOAD_DELAY': 1.5, 
    'USER_AGENT': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) 
    AppleWebKit/537.75.14 (KHTML, ' 
        'like Gecko) Version/7.0.3 Safari/7046A194A', 
    'COOKIES_ENABLED': False 
    } 
    name = "gsmarena_spec" 
    allowed_domains = ["https://www.gsmarena.com/"] 
    start_urls = [ 
     "https://www.gsmarena.com/makers.php3" 
    ] 

    def parse(self, response): 
     # print("Existing settings: %s" % 
     self.settings.attributes.items()) 
     length = len(response.xpath("//table//a").extract()) 
     for i in range(1, length): 
      brand = response.xpath(
      '(//table//a)[{}]/text()'.format(i)).extract()[0] 
      url = "https://www.gsmarena.com/" + \ 
      response.xpath("(//table//a) 
       [{}]/@href".format(i)).extract()[0] 
      yield scrapy.Request(url, 
       callback=self.parse_models_follow_next_page, 
       meta={'brand':brand},dont_filter=True) 

    def parse_models_follow_next_page(self, response): 
     brand = response.meta.get('brand') 
     meta = response.meta 
     doc = html.fromstring(response.body) 
     single_obj = doc.xpath('.//div[@class="makers"]/ul//li') 
     for obj in single_obj: 
      url = self.allowed_domains[0]+obj.xpath('.//a/@href')[0] 
      meta['brand'] = obj.xpath('.//a/@href')[0].split('_')[0] 
      meta['model'] = obj.xpath('.//a/@href')[0] 
      yield scrapy.Request(url=url, callback=self.parse_spec, 
       meta=meta, dont_filter=True) 
     is_next_page = response.xpath(
         "//a[@class=\"pages-next\"]/@href").extract() 
     if is_next_page: 
      next_page = "https://www.gsmarena.com/" + is_next_page[0] 
      yield scrapy.Request(next_page, 
      callback=self.parse_models_follow_next_page, 
      meta={'brand': brand},dont_filter=True) 

    def parse_spec(self, response): 
     item = PhoneItems() 
     meta = response.meta 
     item['model'] = meta['model'] 
     item['brand'] = meta['brand'] 

     #Need to specify details about spec_name 
     # for spec_name, spec in 
     #zip(response.xpath('//table//td[1]').extract(), 
     # response.xpath('//table//td[2]').extract()): 
     #  item[spec_name] = spec 
     yield item