Question

from multiprocessing.dummy import Pool as ThreadPool
class TSNew:
    def __init__(self):
        self.redis_client = redis.StrictRedis(host="172.17.31.147", port=4401, db=0)
        self.pool = ThreadPool(40) # init pool
        self.dnn_model = None
        self.lock=threading.Lock()

        self.t1 = threading.Thread(target=self.load_model_item)
        self.t1.start()

        self.t2 = threading.Thread(target=self.process_user_dict)
        self.t2.start()

    def load_model_item(self):
        while True:
            self.lock.acquire()
            #here  load model from disk as self.dnn_model
            self.lock.release()
            time.sleep(600)
    def predict_memcache(self):
        '''
        process every element in user_dicts
        '''
    def process_user_dict(self,user_dict):
        while True:
            # construct user_dicts as a list
            # use self.dnn_model to predict by self.pool 
            results = self.pool.map(self.predict_memcache, user_dicts)

TSNew_ = TSNew()

def get_user_result():
    logging.info("----------------come in ------------------")
    if request.method == 'POST':
        user_dict_json = request.get_data()# userid
        '''
        ....
        '''
        return 'SUCCESS\n'


@app.route('/', methods=['POST'])
def get_ts_gbdt_id():
    return get_user_result()

from werkzeug.contrib.fixers import ProxyFix
app.wsgi_app = ProxyFix(app.wsgi_app)

if __name__ == '__main__':
    app.run(host='0.0.0.0', port=4444)

类TSNew和process_user_dict中有两个线程使用ThreadPool()通过user_dicts函数并行处理predict_memcache。该函数应使用keras模型，该模型每隔几分钟从磁盘加载一次。如我所写，在加载模型期间（可能需要几秒钟），我不希望模型在predict_memcache中进行预测，因此我尝试使用self.lock.acquire()。但是似乎不起作用，并且在加载时model.predict会给出错误。那么加载时加载模型的正确方法是什么？

如何在multiprocessing.dummy中使用多线程锁定

0 个答案: