有人用过Gevent -1.0.1;请求-2.9.1？我收到以下错误（AttributeError：'module'对象没有属性'epoll'）我相信这种情况最近才开始发生。
另外，如果我使用patch_all（select = False），那么这个错误就会消失，而且并发性也不会发生。
以下是代码：
./greenlet_eg.py
import logging
from datetime import datetime
import functools
from gevent.pool import Pool
import gevent.monkey
gevent.monkey.patch_all() # This must be called before any other modules calling socket/request
from common import ApiClient, work_function, URLS
log = logging.getLogger(__name__)
def main():
t1 = datetime.now()
logging.basicConfig(level=logging.NOTSET, format="%(asctime)s %(levelname)s %(name)s %(process)d/%(thread)d: %(message)s")
no_of_jobs,no_of_workers = 1000,100
log.info('Starting script for # of jobs: {} running # of workers at a time {}...'.format(no_of_jobs, no_of_workers))
api_client = ApiClient(no_of_jobs)
pool = Pool(no_of_workers)
work_function_with_one_arg = functools.partial(work_function, api_client)
results = pool.map(work_function_with_one_arg, api_client.max_urls)
log.info(datetime.now() - t1)
if __name__ == '__main__':
main()
./common.py
import requests
import time
import random
import os
import threading
from datetime import datetime
import functools
import logging
import subprocess
log = logging.getLogger(__name__)
URLS = [
'http://www.python.org',
'http://www.python.org/about/',
'http://www.onlamp.com/pub/a/python/2003/04/17/metaclasses.html',
'http://www.python.org/doc/',
'http://www.python.org/download/',
'http://www.python.org/getit/',
'http://www.python.org/community/',
'https://wiki.python.org/moin/',
'http://planet.python.org/',
'https://wiki.python.org/moin/LocalUserGroups',
'http://www.python.org/psf/',
'http://docs.python.org/devguide/',
'http://www.python.org/community/awards/'
]
class ApiClient():
def __init__(self, no_of_jobs):
self.max_urls = [URLS[random.randint(0,len(URLS)-1)]+str(i) for i in range(no_of_jobs)]
def get_content(self, url):
try:
return requests.get(url).content
except:
logging.exception('Error while getting url {}'.format(url))
return None
def work_function(api_client, url):
log.info('Parsing {}.. PID: {}, threadid: {}'.format(url, os.getpid(), threading.currentThread().ident))
return api_client.get_content(url)
def main():
t1 = datetime.now()
logging.basicConfig(level=logging.NOTSET, format="%(asctime)s %(levelname)s %(name)s %(process)d/%(thread)d: %(message)s")
no_of_jobs,no_of_workers = 1000,100
log.info('Starting script for # of jobs: {} running # of workers at a time {}...'.format(no_of_jobs, no_of_workers))
api_client = ApiClient(no_of_jobs)
results = []
work_function_with_one_arg = functools.partial(work_function, api_client=api_client)
for url in api_client.max_urls:
results.append(work_function_with_one_arg(url=url))
log.info(datetime.now() - t1)
if __name__ == '__main__':
main()
这是错误：...
2017-05-23 16:53:41,501 ERROR root 7757/140199531697968: Error while getting url http://planet.python.org/993
Traceback (most recent call last):
File "/home/barikak/workplace/NetengCommonW/src/NetengCommon/examples/concurrency_vs_threading_vs_multiprocessing/common.py", line 35, in get_content
return requests.get(url).content
File "/brazil-pkg-cache/packages/Requests/Requests-2.x.x.2946.2/RHEL5_64/DEV.STD.PTHREAD/build/lib/python2.7/site-packages/requests/api.py", line 70, in get
return request('get', url, params=params, **kwargs)
File "/brazil-pkg-cache/packages/Requests/Requests-2.x.x.2946.2/RHEL5_64/DEV.STD.PTHREAD/build/lib/python2.7/site-packages/requests/api.py", line 56, in request
return session.request(method=method, url=url, **kwargs)
File "/brazil-pkg-cache/packages/Requests/Requests-2.x.x.2946.2/RHEL5_64/DEV.STD.PTHREAD/build/lib/python2.7/site-packages/requests/sessions.py", line 488, in request
resp = self.send(prep, **send_kwargs)
File "/brazil-pkg-cache/packages/Requests/Requests-2.x.x.2946.2/RHEL5_64/DEV.STD.PTHREAD/build/lib/python2.7/site-packages/requests/sessions.py", line 630, in send
history = [resp for resp in gen] if allow_redirects else []
File "/brazil-pkg-cache/packages/Requests/Requests-2.x.x.2946.2/RHEL5_64/DEV.STD.PTHREAD/build/lib/python2.7/site-packages/requests/sessions.py", line 190, in resolve_redirects
**adapter_kwargs
File "/brazil-pkg-cache/packages/Requests/Requests-2.x.x.2946.2/RHEL5_64/DEV.STD.PTHREAD/build/lib/python2.7/site-packages/requests/sessions.py", line 609, in send
r = adapter.send(request, **kwargs)
File "/brazil-pkg-cache/packages/Requests/Requests-2.x.x.2946.2/RHEL5_64/DEV.STD.PTHREAD/build/lib/python2.7/site-packages/requests/adapters.py", line 423, in send
timeout=timeout
File "/brazil-pkg-cache/packages/Requests/Requests-2.x.x.2946.2/RHEL5_64/DEV.STD.PTHREAD/build/lib/python2.7/site-packages/requests/packages/urllib3/connectionpool.py", line 600, in urlopen
chunked=chunked)
File "/brazil-pkg-cache/packages/Requests/Requests-2.x.x.2946.2/RHEL5_64/DEV.STD.PTHREAD/build/lib/python2.7/site-packages/requests/packages/urllib3/connectionpool.py", line 345, in _make_request
self._validate_conn(conn)
File "/brazil-pkg-cache/packages/Requests/Requests-2.x.x.2946.2/RHEL5_64/DEV.STD.PTHREAD/build/lib/python2.7/site-packages/requests/packages/urllib3/connectionpool.py", line 844, in _validate_conn
conn.connect()
File "/brazil-pkg-cache/packages/Requests/Requests-2.x.x.2946.2/RHEL5_64/DEV.STD.PTHREAD/build/lib/python2.7/site-packages/requests/packages/urllib3/connection.py", line 326, in connect
ssl_context=context)
File "/brazil-pkg-cache/packages/Requests/Requests-2.x.x.2946.2/RHEL5_64/DEV.STD.PTHREAD/build/lib/python2.7/site-packages/requests/packages/urllib3/util/ssl_.py", line 324, in ssl_wrap_socket
return context.wrap_socket(sock, server_hostname=server_hostname)
File "/brazil-pkg-cache/packages/Requests/Requests-2.x.x.2946.2/RHEL5_64/DEV.STD.PTHREAD/build/lib/python2.7/site-packages/requests/packages/urllib3/contrib/pyopenssl.py", line 438, in wrap_socket
rd = util.wait_for_read(sock, sock.gettimeout())
File "/brazil-pkg-cache/packages/Requests/Requests-2.x.x.2946.2/RHEL5_64/DEV.STD.PTHREAD/build/lib/python2.7/site-packages/requests/packages/urllib3/util/wait.py", line 33, in wait_for_read
return _wait_for_io_events(socks, EVENT_READ, timeout)
File "/brazil-pkg-cache/packages/Requests/Requests-2.x.x.2946.2/RHEL5_64/DEV.STD.PTHREAD/build/lib/python2.7/site-packages/requests/packages/urllib3/util/wait.py", line 22, in _wait_for_io_events
with DefaultSelector() as selector:
File "/brazil-pkg-cache/packages/Requests/Requests-2.x.x.2946.2/RHEL5_64/DEV.STD.PTHREAD/build/lib/python2.7/site-packages/requests/packages/urllib3/util/selectors.py", line 364, in __init__
self._epoll = select.epoll()
AttributeError: 'module' object has no attribute 'epoll'
2017-05-23 16:53:41,503 INFO __main__ 7757/140199633593040: 0:00:08.594773
使代码非常简单......
请求2.13.0失败
>>> import sys
>>> import gevent.monkey
>>> gevent.monkey.patch_all() # This must be called before any other modules calling socket/request
>>> import requests
>>>
>>> print 'py version:', sys.version_info
py version: sys.version_info(major=2, minor=7, micro=8, releaselevel='final', serial=0)
>>> print 'requests version:', requests.__version__
requests version: 2.13.0
>>> print 'gevent version:', gevent.__version__
gevent version: 1.0.1
>>>
>>> assert requests.get('http://www.python.org').content
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "/home/barikak/workplace/NetengCommonW/env/NetengCommon-1.0/runtime/lib/python2.7/site-packages/requests/api.py", line 70, in get
return request('get', url, params=params, **kwargs)
File "/home/barikak/workplace/NetengCommonW/env/NetengCommon-1.0/runtime/lib/python2.7/site-packages/requests/api.py", line 56, in request
return session.request(method=method, url=url, **kwargs)
File "/home/barikak/workplace/NetengCommonW/env/NetengCommon-1.0/runtime/lib/python2.7/site-packages/requests/sessions.py", line 488, in request
resp = self.send(prep, **send_kwargs)
File "/home/barikak/workplace/NetengCommonW/env/NetengCommon-1.0/runtime/lib/python2.7/site-packages/requests/sessions.py", line 630, in send
history = [resp for resp in gen] if allow_redirects else []
File "/home/barikak/workplace/NetengCommonW/env/NetengCommon-1.0/runtime/lib/python2.7/site-packages/requests/sessions.py", line 190, in resolve_redirects
**adapter_kwargs
File "/home/barikak/workplace/NetengCommonW/env/NetengCommon-1.0/runtime/lib/python2.7/site-packages/requests/sessions.py", line 609, in send
r = adapter.send(request, **kwargs)
File "/home/barikak/workplace/NetengCommonW/env/NetengCommon-1.0/runtime/lib/python2.7/site-packages/requests/adapters.py", line 423, in send
timeout=timeout
File "/home/barikak/workplace/NetengCommonW/env/NetengCommon-1.0/runtime/lib/python2.7/site-packages/requests/packages/urllib3/connectionpool.py", line 600, in urlopen
chunked=chunked)
File "/home/barikak/workplace/NetengCommonW/env/NetengCommon-1.0/runtime/lib/python2.7/site-packages/requests/packages/urllib3/connectionpool.py", line 345, in _make_request
self._validate_conn(conn)
File "/home/barikak/workplace/NetengCommonW/env/NetengCommon-1.0/runtime/lib/python2.7/site-packages/requests/packages/urllib3/connectionpool.py", line 844, in _validate_conn
conn.connect()
File "/home/barikak/workplace/NetengCommonW/env/NetengCommon-1.0/runtime/lib/python2.7/site-packages/requests/packages/urllib3/connection.py", line 326, in connect
ssl_context=context)
File "/home/barikak/workplace/NetengCommonW/env/NetengCommon-1.0/runtime/lib/python2.7/site-packages/requests/packages/urllib3/util/ssl_.py", line 324, in ssl_wrap_socket
return context.wrap_socket(sock, server_hostname=server_hostname)
File "/home/barikak/workplace/NetengCommonW/env/NetengCommon-1.0/runtime/lib/python2.7/site-packages/requests/packages/urllib3/contrib/pyopenssl.py", line 438, in wrap_socket
rd = util.wait_for_read(sock, sock.gettimeout())
File "/home/barikak/workplace/NetengCommonW/env/NetengCommon-1.0/runtime/lib/python2.7/site-packages/requests/packages/urllib3/util/wait.py", line 33, in wait_for_read
return _wait_for_io_events(socks, EVENT_READ, timeout)
File "/home/barikak/workplace/NetengCommonW/env/NetengCommon-1.0/runtime/lib/python2.7/site-packages/requests/packages/urllib3/util/wait.py", line 22, in _wait_for_io_events
with DefaultSelector() as selector:
File "/home/barikak/workplace/NetengCommonW/env/NetengCommon-1.0/runtime/lib/python2.7/site-packages/requests/packages/urllib3/util/selectors.py", line 364, in __init__
self._epoll = select.epoll()
AttributeError: 'module' object has no attribute 'epoll'
>>>
>>>
请求2.9.1工作
>>> import sys
>>> import gevent.monkey
>>> gevent.monkey.patch_all() # This must be called before any other modules calling socket/request
>>>
>>> import requests
>>>
>>> print 'py version:', sys.version_info
py version: sys.version_info(major=2, minor=7, micro=8, releaselevel='final', serial=0)
>>> print 'requests version:', requests.__version__
requests version: 2.9.1
>>> print 'gevent version:', gevent.__version__
gevent version: 1.0.1
>>>
>>> assert requests.get('http://www.python.org').content
/home/barikak/workplace/NSDSwiftKanbanClientW/env/NSDSwiftKanbanClient-1.0/runtime/lib/python2.7/site-packages/requests/packages/urllib3/util/ssl_.py:120: InsecurePlatformWarning: A true SSLContext object is not available. This prevents urllib3 from configuring SSL appropriately and may cause certain SSL connections to fail. For more information, see https://urllib3.readthedocs.org/en/latest/security.html#insecureplatformwarning.
InsecurePlatformWarning
>>>