Question

我的代码应该比较两个保存为字典的向量（两个pickle文件），并将结果保存到pickle文件中。这工作但非常缓慢。对于一个比较结果，我等待7：2分钟。因为我有很多视频（正好是2033年），这个编程将持续大约10天。这太长了。如何加速Python 2.7的代码？

import math
import csv
import pickle
from itertools import izip

global_ddc_file = 'E:/global_ddc.p'
io = 'E:/AV-Datensatz'
v_source = ''

def dot_product(v1, v2):
    return sum(map(lambda x: x[0] * x[1], izip(v1, v2))) # izip('ABCD', 'xy') --> Ax By

def cosine_measure(v1, v2):
    prod = dot_product(v1, v2)
    len1 = math.sqrt(dot_product(v1, v1))
    len2 = math.sqrt(dot_product(v2, v2))
    if (len1 * len2) <> 0:
        out = prod / (len1 * len2)
    else: out = 0
    return out

def findSource(v):
    v_id = "/"+v[0].lstrip("<http://av.tib.eu/resource/video").rstrip(">")
    v_source = io + v_id
    v_file = v_source + '/vector.p'
    source = [v_id, v_source, v_file]
    return source

def getVector(v, vectorCol):
    with open (v, 'rb') as f:
        try:
            vector_v = pickle.load(f)
        except: print 'file couldnt be loaded'
        tf_idf = []
        tf_idf = [vec[1][vectorCol] for vec in vector_v]
    return tf_idf

def compareVectors(v1, v2, vectorCol):
    v1_source = findSource(v1)
    v2_source = findSource(v2)
    V1 = getVector(v1_source[2], vectorCol)
    V2 = getVector(v2_source[2], vectorCol)
    sim = [v1_source[0], v2_source[0], cosine_measure(V1, V2)]
    return sim

#with open('videos_av_portal_cc_3.0_nur2bspStanford.csv', 'rb') as dataIn:
with open('videos_av_portal_cc_3.0_vollstaendig.csv', 'rb') as dataIn:
#with open('videos_av_portal_cc_3.0.csv', 'rb') as dataIn:
    try:
        reader = csv.reader(dataIn)

        v_source = []
        for row in reader:
            v_source.append(findSource(row))
        #print v_source

        for one in v_source:
            print one[1]
            compVec = []
            for another in v_source:
                if one <> another: 
                    compVec.append(compareVectors(one, another, 3))
            compVec_sort = sorted(compVec, key=lambda cosim: cosim[2], reverse = True) 

            # save vector file for each video
            with open (one[1] + '/compare.p','wb') as f:
                pickle.dump(compVec_sort,f)

    finally:
        dataIn.close()

Answer 1

将代码分为两部分： 1.在向量中加载字典 2.使用多进程multiprocess example比较2个词典 3.根据内存可用性同时启动进程，并在8分钟后结束进程。然后更新第3个字典。 4.然后重新启动下一组数据的过程，按照步骤3继续直到字典长度。

这可以减少总周转时间。如果您需要代码，请告诉我。

Python代码加速

1 个答案: