Question

我正在尝试线程和scrapy。但是，无法弄清楚我做错了什么。只是试图将蜘蛛名称传递给线程类＆amp;得到KeyError。代码如下：

import threading
from PyQt4 import QtCore, QtGui
from scrapy.crawler import CrawlerRunner
from twisted.internet import reactor, defer
from scrapy.utils.project import get_project_settings
from scrapy.utils.log import configure_logging
from my_spider import QuotesSpider



class CrawlerThread(threading.Thread):
    def __init__(self,name):
        super(CrawlerThread,self).__init__() 
        self.name = name
    def stop(self):
        try:
             reactor.stop()
        except:
            pass
    def run(self):
        configure_logging({'LOG_LEVEL': 'DEBUG'})
        crun = CrawlerRunner(get_project_settings())
        crun.crawl(self.name)
        reactor.run(installSignalHandlers=False)


class Form(QtGui.QDialog):


    def __init__(self, parent=None):
        super(Form, self).__init__(parent)
        w =QtGui.QWidget()
        w.resize(250, 150)
        w.move(300, 300)
        self.ct = CrawlerThread(QuotesSpider)
        self.ct.start()

if __name__ == "__main__":
    app = QtGui.QApplication(sys.argv)
    form = Form()
    form.show()


    sys.exit(app.exec_())

错误：

Exception in thread <class 'my_spider.QuotesSpider'>:
Traceback (most recent call last):
  File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner
    self.run()
  File "/home/python/Python/code/Fresh_lostfilm/movies/movies/spiders/GUI.py", line 26, in run
    crun.crawl(self.name)
  File "/usr/local/lib/python2.7/dist-packages/scrapy/crawler.py", line 162, in crawl
    crawler = self.create_crawler(crawler_or_spidercls)
  File "/usr/local/lib/python2.7/dist-packages/scrapy/crawler.py", line 190, in create_crawler
    return self._create_crawler(crawler_or_spidercls)
  File "/usr/local/lib/python2.7/dist-packages/scrapy/crawler.py", line 194, in _create_crawler
    spidercls = self.spider_loader.load(spidercls)
  File "/usr/local/lib/python2.7/dist-packages/scrapy/spiderloader.py", line 51, in load
    raise KeyError("Spider not found: {}".format(spider_name))
KeyError: "Spider not found: <class 'my_spider.QuotesSpider'>"

QuotesSpider来自以下链接： Spiders example

P.s将self.name更改为QuotesSpider直接运行没有任何问题

Answer 1

看起来像更改“名称”到任何其他，不是那么常用的参数名称排序我的问题。以下是编辑后的代码：

import threading
from PyQt4 import QtCore, QtGui
from scrapy.crawler import CrawlerRunner
from twisted.internet import reactor, defer
from scrapy.utils.project import get_project_settings
from scrapy.utils.log import configure_logging
from my_spider import QuotesSpider



class CrawlerThread(threading.Thread):
    def __init__(self,myname): #changing name to myname
    super(CrawlerThread,self).__init__() 
    self.myname = myname       #changing name to myname
    def stop(self):
        try:
             reactor.stop()
        except:
            pass
    def run(self):
        configure_logging({'LOG_LEVEL': 'DEBUG'})
        crun = CrawlerRunner(get_project_settings())
        crun.crawl(self.myname)    #changing name to myname
        reactor.run(installSignalHandlers=False)


class Form(QtGui.QDialog):


    def __init__(self, parent=None):
        super(Form, self).__init__(parent)
        w =QtGui.QWidget()
        w.resize(250, 150)
        w.move(300, 300)
        self.ct = CrawlerThread(QuotesSpider)
        self.ct.start()

if __name__ == "__main__":
    app = QtGui.QApplication(sys.argv)
    form = Form()
    form.show()


    sys.exit(app.exec_())

将蜘蛛名称传递给线程类

1 个答案: