Python中用scrapy框架爬取拉勾网招聘信息

scrapy startproject LaGou

scrapy genspider lagou www.lagou.com

# -*- coding: utf-8 -*-

# Define here the models for your scraped items
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/items.html

import scrapy


class LagouItem(scrapy.Item):
  # define the fields for your item here like:
  job_url = scrapy.Field()
  job_name = scrapy.Field()
  salary = scrapy.Field()
  city = scrapy.Field()
  area = scrapy.Field()
  experience = scrapy.Field()
  education = scrapy.Field()
  labels = scrapy.Field()
  publish_date = scrapy.Field()
  company = scrapy.Field()
  company_feature = scrapy.Field()
  company_public = scrapy.Field()
  company_size= scrapy.Field()

# -*- coding: utf-8 -*-
import scrapy
from LaGou.items import LagouItem
import json
from pprint import pprint
import time


class LagouSpider(scrapy.Spider):
  name = 'lagou'
  allowed_domains = ['www.lagou.com']
  start_urls = ['https://www.lagou.com/jobs/list_python?']

  def __init__(self):
    # 设置头信息, 若不设置的话, 在请求第二页时即被拉勾网认为是爬虫而不能爬取数据
    self.headers = {
      "Accept": "application/json, text/javascript, */*; q=0.01",
      "Connection": "keep-alive",
      "Host": "www.lagou.com",
      "Referer": 'https://www.lagou.com/jobs/list_Python?',
      "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
      "referer": "https://www.lagou.com/jobs/list_python?"
    }
    self.sid = ''
    self.job_url_temp = 'https://www.lagou.com/jobs/{}.html?show={}'
    # 清空文件
    with open('jobs.json', 'w') as f:
      f.truncate()

  def parse(self, response):
    """
    解析起始页
    """
    # response为GET请求的起始页, 自动获取cookie
    # 提交POST带上前面返回的cookies, 访问数据结果第一页
    yield scrapy.FormRequest(
      'https://www.lagou.com/jobs/positionAjax.json?needAddtionalResult=false',
      callback=self.parse_list,
      formdata={"first": "false",
           "pn": "1",
           "kd": "python",
           },
      headers=self.headers
    )
  def parse_list(self, response):
    """
    解析结果列表页的json数据
    """
    # 获取返回的json,转为字典
    res_dict = json.loads(response.text)
    # 判断返回是否成功
    if not res_dict.get('success'):
      print(res_dict.get('msg', '返回异常'))
    else:
      # 获取当前页数
      page_num = res_dict['content']['pageNo']
      print('正在爬取第{}页'.format(page_num))
      # 获取sid
      if not self.sid:
        self.sid = res_dict['content']['showId']
      # 获取响应中的职位url字典
      part_url_dict = res_dict['content']['hrInfoMap']
      # 遍历职位字典
      for key in part_url_dict:
        # 初始化保存职位的item
        item = LagouItem()
        # 拼接完整职位url
        item['job_url'] = self.job_url_temp.format(key, self.sid)
        # 请求职位详情页
        yield scrapy.Request(
          item['job_url'],
          callback=self.parse_detail,
          headers=self.headers,
          meta={'item': item}
        )

      # 获取下一页
      if page_num < 30:
        # time.sleep(2)
        yield scrapy.FormRequest(
          'https://www.lagou.com/jobs/positionAjax.json&#63;needAddtionalResult=false',
          callback=self.parse_list,
          formdata={"first": "false",
               "pn": str(page_num+1),
               "kd": "python",
               "sid": self.sid
               },
          headers=self.headers
        )

  def parse_detail(self, response):
    """
    解析职位详情页
    """
    # 接收item
    item = response.meta['item']
    # 解析数据
    # 获取职位头div
    job_div = response.xpath('//div[@class="position-content-l"]')
    if job_div:
      item['job_name'] = job_div.xpath('./div/h2/text()').extract_first()
      item['salary'] = job_div.xpath('./dd/h4/span[1]/text()').extract_first().strip()
      item['city'] = job_div.xpath('./dd/h4/span[2]/text()').extract_first().strip('/').strip()
      item['area'] = response.xpath('//div[@class="work_addr"]/a[2]/text()').extract_first()
      item['experience'] = job_div.xpath('./dd/h4/span[3]/text()').extract_first().strip('/').strip()
      item['education'] = job_div.xpath('./dd/h4/span[4]/text()').extract_first().strip('/').strip()
      item['labels'] = response.xpath('//ul[@class="position-label clearfix"]/li/text()').extract()
      item['publish_date'] = response.xpath('//p[@class="publish_time"]/text()').extract_first()
      item['publish_date'] = item['publish_date'].split('&')[0]
      # 获取公司dl
      company_div = response.xpath('//dl[@class="job_company"]')
      item['company'] = company_div.xpath('./dt/a/img/@alt').extract_first()
      item['company_feature'] = company_div.xpath('./dd//li[1]/h5[@class="c_feature_name"]/text()').extract_first()
      item['company_feature'] = item['company_feature'].split(',')
      item['company_public'] = company_div.xpath('./dd//li[2]/h5[@class="c_feature_name"]/text()').extract_first()
      item['company_size'] = company_div.xpath('./dd//li[4]/h5[@class="c_feature_name"]/text()').extract_first()
      yield item

from fake_useragent import UserAgent
import random

class RandomUserAgentDM:
  """
  随机获取userAgent
  """
  def __init__(self):
    self.user_agent = UserAgent()

  def process_request(self, request, spider):
    request.headers['User-Agent'] = self.user_agent.random

import json

class LagouPipeline:
  def process_item(self, item, spider):
    with open('jobs.json', 'a', encoding='utf-8') as f:
      item_json = json.dumps(dict(item), ensure_ascii=False, indent=2)
      f.write(item_json)
      f.write('\n')

# 设置日志显示
LOG_LEVEL = 'WARNING'

# 设置ROBOTSTXT协议, 若为true则不能爬取数据
ROBOTSTXT_OBEY = False

# 设置下载器延迟, 反爬虫的一种策略
DOWNLOAD_DELAY = 0.25

# 开启DOWNLOADER_MIDDLEWARES
DOWNLOADER_MIDDLEWARES = {
 # 'LaGou.middlewares.LagouDownloaderMiddleware': 543,
  'LaGou.middlewares.RandomUserAgentDM' :100,
}

# 开启ITEM_PIPELINES
ITEM_PIPELINES = {
 'LaGou.pipelines.LagouPipeline': 300,
}