我试图实时处理子流程日志,在网上进行了一些研究之后,我使用python multithreading和Queue实现了一个版本,我仍然面临一些特殊子流程的死锁问题。有人知道为什么吗?
process = subprocess.Popen(subprocess_cmd_to_run, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stderr_queue = Queue(maxsize=1000)
stderr_enqueue_thread = Thread(target=enqueue_output, args=(process.stderr, stderr_queue))
stderr_enqueue_thread.daemon = True
stderr_euqueue_thread.start()
stdout_queue = Queue(maxsize=1000)
stdout_enqueue_thread = Thread(target=enqueue_output, args=(process.stdout, stdout_queue))
stdout_enqueue_thread.daemon = True
stdout_euqueue_thread.start()
stderr_consumer_thread = Thread(target=consume_output, args=(stderr_queue))
stderr_consumer_thread.daemon = True
stderr_consumer_thread.start()
stdout_consumer_thread = Thread(target=consume_output, args=(stdout_queue))
stdout_consumer_thread.daemon = True
stdout_consumer_thread.start()
stderr_enqueue_thread.join()
stdout_euqueue_thread.join()
stderr_queue.join()
stdout_queue.join()
return_code = process.wait()
def enqueue_output(out, queue):
for line in out:
if line:
decoded_line = line.decode("utf-8")
try:
queue.put_nowait(decoded_line)
except Full:
pass
out.close()
def consume_output(queue):
while True:
line.queue.get()
# do something
queue.task_done()