Question

我有以下parallel_executor.py模块，我用它来同时运行多个进程，

import time
from multiprocessing import Process

class ParallelExecutor(object):
  def __init__(self, pool_size=10):
    self._pool_size = pool_size
    self._processes = []
    self._results = []

def add_task(self, target, args=None, kwargs=None):
    args = [] if not args else args
    kwargs = {} if not kwargs else kwargs
    index = len(self._processes)
    process_args = (index, target, args, kwargs)
    process = Process(target=self._executor, args=process_args)
    self._processes.append(process)
    result = {'result': None, 'end_time': 0, 'completed': False}
    self._results.append(result)
    return index

def run(self, block=True):
    if not block:
      for process in self._processes:
        process.start()
      return None
    else:
      counter = 0
      processes = []
      for process in self._processes:
        processes.append(process)
        process.start()
        if counter >= self._pool_size:

          # Wait for completion and reset counters.
          for i in range(len(processes)):
            processes[i].join()
          processes = []
          counter = 0
          continue
        counter += 1

      # Wait for the left over processes to complete.
      if len(processes) > 0:
        for i in range(len(processes)):
          processes[i].join()
      return self._results



def _executor(self, index, target, args, kwargs):
    try:
      self._results[index]['result'] = target(*args, **kwargs)
      self._results[index]['end_time'] = int(round((time.time())))
      self._results[index]['completed'] = True
    except Exception as exc:
      self._results[index]['exception'] = exc
      self._results[index]['completed'] = True
      raise

我使用它如下（example.py）：

from framework.lib.parallel_executor import ParallelExecutor
import time
import os

def foo(x):
    for i in range(3):
        print x
        time.sleep(0.5)

    return 123

def main():
    runner = ParallelExecutor()
    runner.add_task(foo, ["This"])
    runner.add_task(foo, ["is"])
    runner.add_task(foo, ["a"])
    runner.add_task(foo, ["test"])

    runner.run()
    runner.wait_for_executor_to_finish()
    for i in runner.get_results():
        print i

main()

我的问题是如何使用“foo”的每个语句打印进程ID。通过仅对parallel_executor.py模块进行更改而不触及example.py文件打印到输出，以便稍后我可以执行“grep”＃gre;关于特定过程的输出。

Answer 1

如果不修改示例，你就无法做到这一点，但只需很小的修改就可以达到你想要的效果。

使用Python logging工具，您可以设置默认日志消息，确保每个日志都符合您的标准。

在parallel_executor.py中添加以下内容：

import logging

log_format = "%(process)d: %(message)s"
logging.basicConfig(level=logging.INFO, format=log_format)

在示例中替换以下行：

print x

使用：

logging.info(x)

您会看到您的消息显示为：

34321: message content here

将进程输出记录到单独的日志文件

1 个答案: