检查在Python中打开了哪些文件

Question

我在一段程序中出现错误，该程序应该运行很长时间才会打开太多文件。有什么方法可以跟踪哪些文件已打开，因此我可以偶尔打印该列表并查看问题出在哪里？

Answer 1

我最终在我的程序入口处包装了内置文件对象。我发现我没有关闭我的记录器。

import __builtin__ 
openfiles = set() 
oldfile = __builtin__.file 
class newfile(oldfile): 
    def __init__(self, *args): 
     self.x = args[0] 
     print "### OPENING %s ###" % str(self.x)    
     oldfile.__init__(self, *args) 
     openfiles.add(self) 

    def close(self): 
     print "### CLOSING %s ###" % str(self.x) 
     oldfile.close(self) 
     openfiles.remove(self) 
oldopen = __builtin__.open 
def newopen(*args): 
    return newfile(*args) 
__builtin__.file = newfile 
__builtin__.open = newopen 

def printOpenFiles(): 
    print "### %d OPEN FILES: [%s]" % (len(openfiles), ", ".join(f.x for f in openfiles)) 

Answer 2

我猜你是在泄漏文件描述符。您可能希望查看您的代码以确保您正在关闭所有打开的文件。

Answer 3

在Windows上，您可以使用Process Explorer来显示进程拥有的所有文件句柄。

Answer 4

在Linux上，您可以使用lsof来显示进程打开的所有文件。

Answer 5

在Linux上，你可以看看/proc/self/fd内容：

$ ls -l /proc/self/fd/ 
total 0 
lrwx------ 1 foo users 64 Jan 7 15:15 0 -> /dev/pts/3 
lrwx------ 1 foo users 64 Jan 7 15:15 1 -> /dev/pts/3 
lrwx------ 1 foo users 64 Jan 7 15:15 2 -> /dev/pts/3 
lr-x------ 1 foo users 64 Jan 7 15:15 3 -> /proc/9527/fd 

Answer 6

虽然将打开包装上面的解决方案是为自己的代码很有用，我调试我的客户的第三方库包括一些c扩展代码，所以我需要更直接的方式。达尔文在下面的日常工作，和（我希望）其他类Unix环境：

def get_open_fds(): 
    ''' 
    return the number of open file descriptors for current process 

    .. warning: will only work on UNIX-like os-es. 
    ''' 
    import subprocess 
    import os 

    pid = os.getpid() 
    procs = subprocess.check_output( 
     [ "lsof", '-w', '-Ff', "-p", str(pid) ]) 

    nprocs = len( 
     filter( 
      lambda s: s and s[ 0 ] == 'f' and s[1: ].isdigit(), 
      procs.split('\n')) 
     ) 
    return nprocs 

如果任何人都可以扩展到可以移植到Windows上，我会很感激。

Answer 7

接受的响应有一些限制，因为它似乎没有统计管道。我有一个python脚本，它打开了许多子进程，但未能正确关闭用于通信的标准输入，输出和错误管道。如果我使用已接受的响应，它将无法将这些打开的管道计为打开的文件，但（至少在Linux中）它们是打开的文件并计入打开的文件限制。 sumid和shunc建议的lsof -p解决方案在这种情况下工作，因为它也显示了开放管道。

Answer 8

获取所有打开文件的列表。 handle.exe是微软的Sysinternals Suite的一部分。另一种选择是psutil Python模块，但是我发现“句柄”会打印出更多正在使用的文件。

这是我做的。 Kludgy代码警告。

#!/bin/python3 
# coding: utf-8 
"""Build set of files that are in-use by processes. 
    Requires 'handle.exe' from Microsoft SysInternals Suite. 
    This seems to give a more complete list than using the psutil module. 
""" 

from collections import OrderedDict 
import os 
import re 
import subprocess 

# Path to handle executable 
handle = "E:/Installers and ZIPs/Utility/Sysinternalssuite/handle.exe" 

# Get output string from 'handle' 
handle_str = subprocess.check_output([handle]).decode(encoding='ASCII') 

""" Build list of lists. 
    1. Split string output, using '-' * 78 as section breaks. 
    2. Ignore first section, because it is executable version info. 
    3. Turn list of strings into a list of lists, ignoring first item (it's empty). 
""" 
work_list = [x.splitlines()[1:] for x in handle_str.split(sep='-' * 78)[1:]] 

""" Build OrderedDict of pid information. 
    pid_dict['pid_num'] = ['pid_name','open_file_1','open_file_2', ...] 
""" 
pid_dict = OrderedDict() 
re1 = re.compile("(.*?\.exe) pid: ([0-9]+)") # pid name, pid number 
re2 = re.compile(".*File.*\s\s\s(.*)") # File name 
for x_list in work_list: 
    key = '' 
    file_values = [] 
    m1 = re1.match(x_list[0]) 
    if m1: 
     key = m1.group(2) 
#  file_values.append(m1.group(1)) # pid name first item in list 

    for y_strings in x_list: 
     m2 = re2.match(y_strings) 
     if m2: 
      file_values.append(m2.group(1)) 
    pid_dict[key] = file_values 

# Make a set of all the open files 
values = [] 
for v in pid_dict.values(): 
    values.extend(v) 
files_open = sorted(set(values)) 

txt_file = os.path.join(os.getenv('TEMP'), 'lsof_handle_files') 

with open(txt_file, 'w') as fd: 
    for a in sorted(files_open): 
     fd.write(a + '\n') 
subprocess.call(['notepad', txt_file]) 
os.remove(txt_file) 

Answer 9

正如前面所说，你可以列出Linux上FDS在的/ proc /自/依存，这里有一个简单的方法以编程方式一一列举：

import os 
import sys 
import errno 

def list_fds(): 
    """List process currently open FDs and their target """ 
    if sys.platform != 'linux2': 
     raise NotImplementedError('Unsupported platform: %s' % sys.platform) 

    ret = {} 
    base = '/proc/self/fd' 
    for num in os.listdir(base): 
     path = None 
     try: 
      path = os.readlink(os.path.join(base, num)) 
     except OSError as err: 
      # Last FD is always the "listdir" one (which may be closed) 
      if err.errno != errno.ENOENT: 
       raise 
     ret[int(num)] = path 

    return ret 

Answer 10

要列出所有打开的文件一个跨平台的方式，我会推荐psutil。

#!/usr/bin/env python 
import psutil 

for proc in psutil.process_iter(): 
    print proc.open_files() 

原始问题隐含地限制了操作到当前正在运行的进程，可以通过psutil的Process类访问该进程。

proc = psutil.Process() 
print proc.open_files() 

最后，你要使用的帐户具有相应权限来访问这些信息来运行代码或您可能会看到存取遭拒错误。

Answer 11

您可以使用以下脚本。它建立在克劳迪乌的answer上。它解决了一些问题，并增加了额外的功能：

• 打印该文件被打开的程序退出
• 打印的堆栈跟踪
• 关键字参数支持

下面的代码和指向gist的链接，该链接可能更新。

""" 
Collect stacktraces of where files are opened, and prints them out before the 
program exits. 

Example 
======== 

monitor.py 
---------- 
from filemonitor import FileMonitor 
FileMonitor().patch() 
f = open('/bin/ls') 
# end of monitor.py 

$ python monitor.py 
    ---------------------------------------------------------------------------- 
    path = /bin/ls 
    > File "monitor.py", line 3, in <module> 
    >  f = open('/bin/ls') 
    ---------------------------------------------------------------------------- 

Solution modified from: 
https://stackoverflow.com/questions/2023608/check-what-files-are-open-in-python 
""" 
from __future__ import print_function 
import __builtin__ 
import traceback 
import atexit 
import textwrap 


class FileMonitor(object): 

    def __init__(self, print_only_open=True): 
     self.openfiles = [] 
     self.oldfile = __builtin__.file 
     self.oldopen = __builtin__.open 

     self.do_print_only_open = print_only_open 
     self.in_use = False 

     class File(self.oldfile): 

      def __init__(this, *args, **kwargs): 
       path = args[0] 

       self.oldfile.__init__(this, *args, **kwargs) 
       if self.in_use: 
        return 
       self.in_use = True 
       self.openfiles.append((this, path, this._stack_trace())) 
       self.in_use = False 

      def close(this): 
       self.oldfile.close(this) 

      def _stack_trace(this): 
       try: 
        raise RuntimeError() 
       except RuntimeError as e: 
        stack = traceback.extract_stack()[:-2] 
        return traceback.format_list(stack) 

     self.File = File 

    def patch(self): 
     __builtin__.file = self.File 
     __builtin__.open = self.File 

     atexit.register(self.exit_handler) 

    def unpatch(self): 
     __builtin__.file = self.oldfile 
     __builtin__.open = self.oldopen 

    def exit_handler(self): 
     indent = ' > ' 
     terminal_width = 80 
     for file, path, trace in self.openfiles: 
      if file.closed and self.do_print_only_open: 
       continue 
      print("-" * terminal_width) 
      print(" {} = {}".format('path', path)) 
      lines = ''.join(trace).splitlines() 
      _updated_lines = [] 
      for l in lines: 
       ul = textwrap.fill(l, 
            initial_indent=indent, 
            subsequent_indent=indent, 
            width=terminal_width) 
       _updated_lines.append(ul) 
      lines = _updated_lines 
      print('\n'.join(lines)) 
      print("-" * terminal_width) 
      print()