使用python中的子进程重定向标准输出速度非常慢

Question

当我在Python中使用子进程重定向标准输出时，吞吐量非常低。我做错了吗？

基本上，我管道外部程序的标准输出放入队列中。然后在另一个功能中，我将它打印在控制台中。

这里是hexdump都一个示例代码来生成随机输出：

from subprocess import Popen, PIPE 
from queue import Queue 
import sys 
from threading import Thread, Event 
import threading 

class Buffer(Queue): 

    def __init__(self, *args, **kwargs): 
     Queue.__init__(self, *args, **kwargs) 

    def write(self, line): 
     self.put_nowait(line) 
     self.join() 

    def read(self): 
     element = self.get_nowait() 
     self.task_done() 
     return element 

def write_output(buffer, stopped): 

    hexdump = Popen(['hexdump', '-C', '/dev/urandom'], stdout=PIPE) 
    while hexdump.returncode is None: 
     for line in hexdump.stdout.readlines(8192): 
      buffer.write(line) 
      if stopped.is_set(): 
       hexdump.terminate() 
       hexdump.wait() 
       print('process terminated.') 
       break 

def read_output(buffer, stopped): 
    while not stopped.is_set(): 
     while not buffer.empty(): 
      output = buffer.read() 
      print('********* output: {}'.format(output)) 
      sys.stdout.flush() 
    print('stopped') 
    sys.stdout.flush() 


buffer = Buffer() 
stopped = Event() 


generate_random_output = Thread(target=write_output, args=(buffer, stopped)) 
generate_random_output.name = 'generate_random_output' 
generate_random_output.start() 

process_output = Thread(target=read_output, args=(buffer, stopped)) 
process_output.name = 'process_output' 
process_output.start() 

try: 
    while True: 
     continue 
except KeyboardInterrupt: 
    stopped.set() 
    generate_random_output.join() 
    process_output.join() 
    print('finished generating') 
    print('finished processing') 

我希望得到任何帮助。

Answer 1

不是重定向你的输出队列 - 直接过程是：

def write_output(buffer, stopped): 

    hexdump = Popen(['hexdump', '-C', '/dev/urandom'], stdout=PIPE) 
    while hexdump.poll() is None: 
     while not stopped.is_set(): 
      for line in iter(hexdump.stdout.readline, b''): 
       print('********* output: %s' % line.decode(), end='') 
       sys.stdout.flush() 

     hexdump.terminate() 
     hexdump.wait() 
     print('process terminated.') 
     break