python爬虫方法实例分析

import requestsres=requests.get('http://bj.xiaozhu.com/')print(res)#返回结果为<Response [200]>，说明请求网址成功，若为404，400则请求网址失败print(res.text)

import requestsheaders={}res=requests.get('http://bj.xiaozhu.com/',headers=headers)print(res.text)

import requestsheaders={}res=requests.get('http://bj.xiaozhu.com/',headers=headers)try:    print(res.text)except ConnectionError:    print('拒绝连接')

import requestsfrom bs4 import BeautifulSoupheaders={}res=requests.get('http://bj.xiaozhu.com/',headers=headers)soup=BeautifulSoup(res.text,'html.parser')print(soup.prettify())

import requestsfrom bs4 import BeautifulSoupheaders={) Chrome/73.0.3683.86 Safari/537.36'}res=requests.get('http://bj.xiaozhu.com/',headers=headers)res.encoding = 'utf-8'soup=BeautifulSoup(res.text,'html.parser')#price=soup.select('#page_list > ul > li:nth-child(1) > div.result_btm_con.lodgeunitname > div:nth-child(1) > span > i')#price=soup.select('#page_list > ul > li:nth-of-type(1) > div.result_btm_con.lodgeunitname > div:nth-of-type(1) > span > i')prices=soup.select('#page_list > ul > li > div.result_btm_con.lodgeunitname > div > span > i')print(prices)for price in prices:    print (price) #获取单条html信息    print (price.get_text()) #获取中间文字信息

from bs4 import BeautifulSoupimport pandas as pdimport requestsimport timeimport lxml#加入请求头headers={}#定义判断用户性别的函数def judgment_sex(class_name):    if class_name ==['member_icol']:        return '女'    else:        return '男'#定义获取详细页URL的函数def get_links(url):    wb_data=requests.get(url,headers=headers)    #print(wb_data)    soup=BeautifulSoup(wb_data.text,'lxml')    #print(soup)    links=soup.select('#page_list > ul > li > a') #links为url列表                      #page_list > ul > li:nth-child(1) > a    for link in links:        href=link.get("href")        get_info(href) #循环出的url,依次调用get_info()函数#定义获取网页信息的函数def get_info(url):    wb_data=requests.get(url,headers=headers)    soup=BeautifulSoup(wb_data.text,'lxml')    #tittles=soup.select('div.pho_info>h5')    #tittles=soup.select('#page_list > ul > li> div.result_btm_con.lodgeunitname > div.result_intro > a > span')                         tittles = soup.select('div.pho_info > h5')    addresses = soup.select('span.pr5')    prices = soup.select('#pricePart > div.day_l > span')    imgs = soup.select('#floatRightBox > div.js_box.clearfix > div.member_pic > a > img')    names = soup.select('#floatRightBox > div.js_box.clearfix > div.w_240 > h7 > a')    sexs = soup.select('#floatRightBox > div.js_box.clearfix > div.member_pic > div')    for tittle,address,price,img,name,sex in zip(tittles,addresses,prices,imgs,names,sexs):        data={              'tittle':tittle.get_text().strip(),              'address':address.get_text().strip(),              'price':price.get_text(),              'img':img.get("src"),              'name':name.get_text(),              'sex':judgment_sex(sex.get("class"))              }    #print(data)#获取信息并通过字典的信息打印    #print(pd.DataFrame([data]).head())if __name__ == '__main__':    urls=['http://bj.xiaozhu.com/search-duanzufang-p{}-0/'.format(number) for number in range(1,2)]    for single_url in urls:        print(single_url)        get_links(single_url)        time.sleep(2)