import Queue
import threading
import urllib2
import time
from BeautifulSoup import BeautifulSoup
hosts = ["http://yahoo.com", "http://google.com", "http://amazon.com",
"http://ibm.com", "http://apple.com"]
queue = Queue.Queue()
out_queue = Queue.Queue()
class ThreadUrl(threading.Thread):
"""Threaded Url Grab"""
def __init__(self, queue, out_queue):
threading.Thread.__init__(self)
self.queue = queue
self.out_queue = out_queue
def run(self):
while True:
#grabs host from queue
host = self.queue.get()
#grabs urls of hosts and then grabs chunk of webpage
url = urllib2.urlopen(host)
chunk = url.read()
#place chunk into out queue
self.out_queue.put(chunk)
#signals to queue job is done
self.queue.task_done()
class DatamineThread(threading.Thread):
"""Threaded Url Grab"""
def __init__(self, out_queue):
threading.Thread.__init__(self)
self.out_queue = out_queue
def run(self):
while True:
#grabs host from queue
chunk = self.out_queue.get()
#parse the chunk
soup = BeautifulSoup(chunk)
print soup.findAll(['title'])
#signals to queue job is done
self.out_queue.task_done()
start = time.time()
def main():
#spawn a pool of threads, and pass them queue instance
for i in range(5):
t = ThreadUrl(queue, out_queue)
t.setDaemon(True)
t.start()
#populate queue with data
for host in hosts:
queue.put(host)
for i in range(5):
dt = DatamineThread(out_queue)
dt.setDaemon(True)
dt.start()
#wait on the queue until everything has been processed
queue.join()
out_queue.join()
main()
print "Elapsed Time: %s" % (time.time() - start)
有时候,我得到这个错误在这里:
异常螺纹加工-10(翻译关闭期间最有可能上调)
解释请什么原因造成的。
更新由其他作者:
下面是完整的例外,我在类似的代码,请参阅:
Exception in thread Thread-1 (most likely raised during interpreter shutdown):
Traceback (most recent call last):
File "/usr/local/lib/python2.7/threading.py", line 552, in __bootstrap_inner
File "/usr/local/lib/python2.7/threading.py", line 505, in run
File "mine.py", line 86, in run
File "/usr/local/lib/python2.7/Queue.py", line 168, in get
File "/usr/local/lib/python2.7/threading.py", line 237, in wait
<type 'exceptions.TypeError'>: 'NoneType' object is not callable
一些想法人们可能更愿意帮助你:1.解释你已经做了什么来诊断问题以及你的努力失败的原因。 2.尝试将代码减少到发生错误所需的最小值。 3.给出错误的完整追溯。 –
我在类似的代码中看到相同的错误。我会在上面添加我的完整例外。 –
我在使用multiprocessing.Pool类时偶尔会发现此错误。它似乎来自它创建的守护线程,当解释器在关闭期间开始删除事物时,它仍然可以运行。一个解决方案,如果它没有解决它,它已经减少了我的观察门限以下的频率,当你完成多处理时,调用pool.terminate()或类似的方法。 –