Question

我最近看到了关于如何functions are objects in python的回答/评论。所以，我想知道为什么当我拿这个例子，并在初始化变量时围绕它创建一个类时，它的工作方式不同。（类示例收到酸洗错误）： PicklingError: Can't pickle <type 'instancemethod'>: attribute lookup __builtin__.instancemethod failed

有谁知道这是为什么？

链接中的示例代码：

import multiprocessing as mp

def f(x):
    f.q.put('Doing: ' + str(x))
    return x*x

def f_init(q):
    f.q = q

def main():
    jobs = range(1,6)

    q = mp.Queue()
    p = mp.Pool(None, f_init, [q])
    results = p.imap(f, jobs)
    p.close()

    for i in range(len(jobs)):
        print q.get()
        print results.next()

if __name__ == '__main__':
    main()

将put f放入类中时的相同示例：

import multiprocessing as mp

class F:
    def __init__(self, q):
        self.q = q
    def f(x):
        self.q.put('Doing: ' + str(x))
        return x*x

def main():
    jobs = range(1,6)

    q = mp.Queue()
    p = mp.Pool(None)
    f = F(q)
    results = p.imap(f.f, jobs)
    p.close()

    for i in range(len(jobs)):
        print q.get()
        print results.next()

if __name__ == '__main__':
    main()

Answer 1

实例方法不能自动选择。所以

p.imap(f.f, jobs)

失败，因为p.imap试图挑选参数。有一种方法可以“教”泡菜how to pickle instance methods（见Steven Bethard的回答），但是您的代码还有另一个问题：将队列传递给实例会导致RuntimeError：

RuntimeError: Queue objects should only be shared between processes through inheritance

错误消息有点令人困惑（至少对我来说），因为可以将队列作为参数传递给p.imap，但是你无法将它传递给类{{首先，然后通过F将其传输到工作进程。

无论如何，由于这些问题，我建议坚持原始代码，而不是试图将代码包装在一个类中。

以下是如何挑选实例方法的示例：

f.f

产量

import multiprocessing as mp
import copy_reg
import types

def _pickle_method(method):
    # Author: Steven Bethard (author of argparse)
    # http://bytes.com/topic/python/answers/552476-why-cant-you-pickle-
    # instancemethods
    func_name = method.im_func.__name__
    obj = method.im_self
    cls = method.im_class
    cls_name = ''
    if func_name.startswith('__') and not func_name.endswith('__'):
        cls_name = cls.__name__.lstrip('_')
    if cls_name:
        func_name = '_' + cls_name + func_name
    return _unpickle_method, (func_name, obj, cls)


def _unpickle_method(func_name, obj, cls):
    # Author: Steven Bethard
    # http://bytes.com/topic/python/answers/552476-why-cant-you-pickle-
    # instancemethods
    for cls in cls.mro():
        try:
            func = cls.__dict__[func_name]
        except KeyError:
            pass
        else:
            break
    return func.__get__(obj, cls)

# This call to copy_reg.pickle allows you to pass methods as the first arg to
# mp.Pool methods. If you comment out this line, `pool.map(self.foo, ...)` results in
# PicklingError: Can't pickle <type 'instancemethod'>: attribute lookup
# __builtin__.instancemethod failed

copy_reg.pickle(types.MethodType, _pickle_method, _unpickle_method)

class F(object):
    def f(self, x):
        fq.put('Doing: ' + str(x))        
        return x*x

def f_init(q):
    # http://stackoverflow.com/a/3843313/190597 (Olson)
    global fq
    fq = q

def main():
    jobs = range(1,6)
    q = mp.Queue()
    p = mp.Pool(None, f_init, [q]) 
    f = F()
    results = p.imap(f.f, jobs)
    p.close()

    for r in results:
        print(r, q.get())

if __name__ == '__main__':
    main()

函数初始化和对象初始化（多处理）

1 个答案: