Question

我正在尝试使用python的scrapy框架构建一个蜘蛛来刮取纽约理工学院的课程数据......以下是我的蜘蛛（nyitspider.py）。有人可以告诉我哪里出错了。

from scrapy.spiders import CrawlSpider, Rule, BaseSpider, Spider
from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor
from scrapy.selector import Selector
from scrapy.http import HtmlResponse

from nyit_sample.items import NyitSampleItem


class nyitspider(CrawlSpider):
name = 'nyitspider'
allowed_domains = ['nyit.edu']
start_urls = ['http://www.nyit.edu/academics/courses/']

rules = (
    Rule(LxmlLinkExtractor(
         allow=('.*/academics/courses', ),
    )),

Rule(LxmlLinkExtractor(
         allow=('.*/academics/courses/[a-z][a-z][a-z]-[a-z][a-z]-[0-9][0-9]    [0-9]/', ),
    ), callback='parse_item'),

)

def parse_item(self, response):
    item = Course()
    item["institute"] = 'New York Institute of Technology'
    item['site'] = 'www.nyit.edu'
    item['title'] = response.xpath('//*[@id="course_catalog_table"]/tbody/tr[1]/td[2]/a').extract()[0]
item['id'] = response.xpath('//*[@id="course_catalog_table"]/tbody/tr[1]/td[1]/a').extract()[0]
    item['credits'] = response.xpath('//*[@id="course_catalog_table"]/tbody/tr[1]/td[3]').extract()[0]
    item['description'] = response.xpath('//*[@id="course_catalog_table"]/tbody/tr[2]/td/text()[1]').extract()[0]



    yield item

Answer 1

您必须在parse_item方法中正确声明该项，并且该方法应返回一些内容。这是一个建议，但你必须改进它：

# -*- coding: utf-8 -*-
from scrapy.spiders import CrawlSpider, Rule, BaseSpider, Spider
from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor
from scrapy.selector import Selector
from scrapy.http import HtmlResponse

from nyit_sample.items import NyitSampleItem


class nyitspider(CrawlSpider):
    name = 'nyitspider'
    allowed_domains = ['nyit.edu']
    start_urls = ['http://www.nyit.edu/academics/courses/']

    rules = (
        Rule(LxmlLinkExtractor(
             allow=('.*/academics/courses', ),
        ), callback='parse_item'),   
        Rule(LxmlLinkExtractor(
             allow=('.*/academics/courses/[a-z][a-z][a-z]-[a-z][a-z]-[0-9][0-9]    [0-9]/', ),
        ), callback='parse_item'),

    )

    def parse_item(self, response):
        item = NyitSampleItem()
        item['institute'] = 'New York Institute of Technology'
        item['site'] = 'www.nyit.edu'
        item['title'] = response.xpath('string(//*[@id="course_catalog_table"]/tbody/tr[1]/td[2]/a)').extract()[0]
        item['id'] = response.xpath('string(//*[@id="course_catalog_table"]/tbody/tr[1]/td[1]/a)').extract()[0]
        item['credits'] = response.xpath('string(//*[@id="course_catalog_table"]/tbody/tr[1]/td[3])').extract()[0]
        item['description'] = response.xpath('//*[@id="course_catalog_table"]/tbody/tr[2]/td/text()[1]').extract()[0]
        return item

Crawlspider规则不起作用

1 个答案: