python中怎么实现一个多线程爬虫

mkdir mutiThreadCrawiercd  mutiThreadCrawiermkdir content #存爬下来的页面virtualenv env --python =python3.6 #创建虚拟环境source env/bin/activate   #使虚拟环境生效

import timeimport reimport threadingimport urllibimport requestsfrom bs4 import BeautifulSoup

g_mutex = threading.Condition()  # 可以加锁/释放锁print(g_mutex)print(type(g_mutex))g_urls = []  # 存放解析出来的url对应的网页源码g_queue_urls = [] # 待爬取的urlg_exist_urls = []  # 已经爬过的urlg_failed_urls = [] # 失败的链接g_total_count = 0  # 已经下载的页面的计数器

class CrawlerThread(threading.Thread):    def __init__(self,url,filename,tid):        threading.Thread.__init__(self)        self.filename=filename        self.url =url        self.tid=tid    def run(self):        try:            resp=urllib.request.urlopen(self.url)            html=resp.read()            with open('content/'+self.filename,'wb') as f:                f.write(html)        except Exception as e:            g_exist_urls.append(self.url)            g_failed_urls.append(self.url)            print(f'页面{self.url}下载失败！')        g_mutex.acquire()        g_urls.append(html)        g_exist_urls.append(self.url)        g_mutex.release()

class Crawler:    def __init__(self,name,domain,thread_number):        self.name=name        self.domain=domain        self.thread_number=thread_number        self.logfile=open('log.txt','w')        self.thread_pool=[]        self.url = 'http://'+domain     def spider(self):# 内容会随着爬虫的进行而更新        global g_queue_urls# 初始，队列中仅有一个url        g_queue_urls.append(self.url)# 爬取的深度        depth =0        print(f'爬虫{self.name}开始启动........')        while g_queue_urls:            depth +=1            print(f'当前爬取深度是{depth}')            self.logfile.write(f'URL:{g_queue_urls[0]}')            self.download_all() # 下载所有            self.update_queque_url() # 更新 url队列            self.logfile.write(f">>>Depth:{depth}")            count = 0            while count <len(g_queue_urls):                self.logfile.write(f"累计爬取{g_total_count}条，爬取是第{g_queue_urls[count]}个")                count+=1    def download_all(self):        global g_queue_urls        global g_total_count        i=0        while i < len(g_queue_urls):            j=0            while j<self.thread_number and i+j <len(g_queue_urls):                g_total_count +=1                print(g_queue_urls[i+j])                thread_result=self.download(g_queue_urls[i+j],f"{g_total_count}.html",j)                if thread_result  is not None:                      print(f'线程{i+j}启动')                j +=1            i=i+j            for thread in self.thread_pool:                thread.join(25)        g_queue_urls=[]    def download(self,url,filename,tid):        print(url,filename,tid)        creawler_thread= CrawlerThread(url,filename,tid)        self.thread_pool.append(creawler_thread)        creawler_thread.start()    def update_queque_url(self):        global g_queue_urls        global g_exist_urls#已经爬过的url        new_urls=[]#新发现的url        for url_content in g_urls:            new_urls +=self.get_Url(url_content)# 从页面中提取新url        g_queue_urls=list(set(new_urls)  -set(g_exist_urls)) # 去除重复的和已经爬过的    def get_Url(self,content):    	'''        从网页源代码中提取url        '''        links =[] # 保存提取到的href        try:            soup =BeautifulSoup(content)            for link in soup.findAll('a'):                if link is not None and link.get('href') is not None:                    if self.domain in link['href']: 			# 如果link是本网站的绝对地址                        links.append(link)                    elif  len(link['href']) >10 and  'http://' not in link['href']:				 # 如果link是相对地址                        links.append(self.url +link['href'])        except Exception as e:            print("fail to get url",e)        return links

if __name__=="__main__":    domain ="www.geyanw.com"    thread_number=10    name="geyan"    crawler =Crawler(name,domain,thread_number)    crawler.spider()