2017-02-10 77 views
0
import requests 
from requests import Session 
from bs4 import BeautifulSoup 
import re 
from multiprocessing.dummy import Pool as ThreadPool 

def get_total_pages(): 
    tut = [] 
    base_url = 'Your group ' 
    for url in [base_url % i for i in range(1, 27)]: 
     tut.append(url) 
    print(tut) 
    #get_data_from_page(tut) 
    pool = ThreadPool(8) 
    results = pool.map(get_data_from_page, tut) 

def get_data_from_page(tut): 
    f = open("emails.txt", 'a') 
    email = [] 
    for a in tut: 
     link = s.get(a).text 
     soup = BeautifulSoup(link, 'lxml') 
     links = soup.find('div', class_="mens").find_all('span', class_="inviz") 
     for e in links: 
      emails = e.text 
      f.write(emails + ', ') 
      email.append(emails) 
    print(email) 

def main(): 
    get_total_pages() 
if __name__ == '__main__': 
    main() 

这将导致一个错误,说它只适用于多工作,以及:请求例外,只有multiprocessing.dummy导入池

raise MissingSchema(error) 
requests.exceptions.MissingSchema: Invalid URL 'h': No schema supplied. Perhaps you meant http://h? 
+0

您能否快速解释您为什么要这么做?由于您是垃圾邮件发送者试图从互联网上收集电子邮件地址 – hansaplast

+0

修复缩进,易读性 – mirabilos

回答

1

的问题是在这

for a in tut: 
    link = s.get(a).text 

和需要

只是

link = s.get(a).text 
#without for