多处理 - pyodbc IOError:错误的消息长度

时间:2016-02-12 00:05:40

标签: python-2.7 pyodbc python-multiprocessing

尝试在多个进程之间共享pyodbc连接时,我意外收到IOError: bad message length错误,尤其是当N超过4(核心数)时。有时我也会将cPickle.UnpicklingError: invalid load key, '#'.pyodbc.ProgrammingError: ('24000', '[24000] [FreeTDS][SQL Server]Invalid cursor state (0) (SQLExecDirectW)')视为错误。

# Import custom python packages
import multiprocessing
import multiprocessing.managers as mm
import pathos.multiprocessing as mp
import pyodbc, datetime, time

class MyConn(object):
    def __init__(self):
        self.conn = None
        self.cursor = None

    def connect_to_db(self):
        self.conn = pyodbc.connect("DSN=cpmeast;UID=dntcore;PWD=dntcorevs2")
        self.cursor = self.conn.cursor()

    def run_qry(self, data):
        print 'Running query', data
        self.cursor.execute("WAITFOR DELAY '00:00:01';select GETDATE(), '"+str(data)+"';")
        l = self.cursor.fetchall()
        _l = []
        for i in l:
            _l.append(list(i))
        print 'Result for query', data, _l
        return _l


class MyManagerClass(object):
    def __init__(self):
        self.result = multiprocessing.Manager().list()

    def read_data(self, *args):
        conn = args[0][0]
        data = args[0][1]
        l = conn.run_qry(data)
        self.result.append(l)

class MyManager(mm.BaseManager):
    pass # Pass is really enough. Nothing needs to be done here.

def main():
    time_start = time.time()
    MyManager.register("MyConn", MyConn)
    manager = MyManager()
    manager.start()
    a = manager.MyConn()
    a.connect_to_db()

    dbm = MyManagerClass()

    pool = mp.ProcessingPool(4)
    jobs = []
    N = 5
    for i in range(N):
        jobs.append((a, str(i)))
    for i in pool.imap(dbm.read_data, jobs):
        print 'result'
    pool.close()
    pool.join()

    print 'Result', dbm.result
    print 'Closed'
    time_stop = time.time()
    msg = 'runtime: {0}'.format(str(datetime.timedelta
                                             (seconds=time_stop-time_start)))
    print msg

if __name__ == '__main__':
    main()

0 个答案:

没有答案