pip install scrapy

scrapy startproject questions

# -*- coding: utf-8 -*-
import scrapy
import time
import numpy
import re
from questions.items import QuestionsItem
class xueersiSpider(scrapy.Spider):
  name = "xueersi" # 爬蟲名字
  allowed_domains = ["tiku.xueersi.com"] # 目標(biāo)的域名
　　# 爬取的目標(biāo)地址
  start_urls = [
    "http://tiku.xueersi.com/shiti/list_1_1_0_0_4_0_1",
    "http://tiku.xueersi.com/shiti/list_1_2_0_0_4_0_1",
    "http://tiku.xueersi.com/shiti/list_1_3_0_0_4_0_1",
  ]
  levels = ['偏易','中檔','偏難']
  subjects = ['英語','語文','數(shù)學(xué)']
　　 # 爬蟲開始的時(shí)候，自動(dòng)調(diào)用該方法，如果該方法不存在會(huì)自動(dòng)調(diào)用parse方法
  # def start_requests(self):
  #   yield scrapy.Request('http://tiku.xueersi.com/shiti/list_1_2_0_0_4_0_39',callback=self.getquestion)
　　# start_requests方法不存在時(shí)，parse方法自動(dòng)被調(diào)用
  def parse(self, response):
　　　　　# xpath的選擇器語法不多介紹，可以直接查看官方文檔
    arr = response.xpath("http://ul[@class='pagination']/li/a/text()").extract()
    total_page = arr[3]
　　　　　# 獲取分頁
    for index in range(int(total_page)):
      yield scrapy.Request(response.url.replace('_0_0_4_0_1',"_0_0_4_0_"+str(index)),callback=self.getquestion) # 發(fā)出新的請(qǐng)求，獲取每個(gè)分頁所有題目
  # 獲取題目
  def getquestion(self,response):
    for res in response.xpath('//div[@class="main-wrap"]/ul[@class="items"]/li'):
      item = QuestionsItem() # 實(shí)例化Item類
      # 獲取問題
      questions = res.xpath('./div[@class="content-area"]').re(r'<div class="content-area">?([\s\S]+?)<(table|\/td|div|br)')
      if len(questions):
        # 獲取題目
        question = questions[0].strip()
        item['source'] = question
        dr = re.compile(r'<[^>]+>',re.S)
        question = dr.sub('',question)
        content = res.extract()
        item['content'] = question
        # 獲取課目
        subject = re.findall(ur'http:\/\/tiku\.xueersi\.com\/shiti\/list_1_(\d+)',response.url)
        item['subject'] = self.subjects[int(subject[0])-1]
        # 獲取難度等級(jí)
        levels = res.xpath('//div[@class="info"]').re(ur'難度：([\s\S]+?)<')
        item['level'] = self.levels.index(levels[0])+1
        
        # 獲取選項(xiàng)
        options = re.findall(ur'[A-D][\.．]([\s\S]+?)<(\/td|\/p|br)',content)
        item['options'] = options
        if len(options):
          url = res.xpath('./div[@class="info"]/a/@href').extract()[0]
          request = scrapy.Request(url,callback=self.getanswer)
          request.meta['item'] = item # 緩存item數(shù)據(jù)，傳遞給下一個(gè)請(qǐng)求
          yield request
      #for option in options:
  # 獲取答案      
  def getanswer(self,response):
    
    res = response.xpath('//div[@class="part"]').re(ur'<td>([\s\S]+?)<\/td>')
    con = re.findall(ur'([\s\S]+?)<br>[\s\S]+?([A-D])',res[0]) # 獲取含有解析的答案
    if con:
      answer = con[0][1]
      analysis = con[0][0] # 獲取解析
    else:
      answer = res[0]
      analysis = ''
    if answer:
      item = response.meta['item'] # 獲取item
      item['answer'] = answer.strip()
      item['analysis'] = analysis.strip()
      item['answer_url'] = response.url
      yield item # 返回item,輸出管道（pipelines.py）會(huì)自動(dòng)接收該數(shù)據(jù)
 

# -*- coding: utf-8 -*-
# Define here the models for your scraped items
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/items.html
import scrapy
class QuestionsItem(scrapy.Item):
  content = scrapy.Field()
  subject = scrapy.Field()
  level = scrapy.Field()
  answer = scrapy.Field()
  options = scrapy.Field()
  analysis = scrapy.Field()
  source = scrapy.Field()
  answer_url = scrapy.Field()
  pass

# -*- coding: utf-8 -*-
# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
import pymysql
import md5
class QuestionsPipeline(object):
  def __init__(self): 
    # 建立數(shù)據(jù)庫連接 
    self.connect = pymysql.connect('localhost','root','','question',use_unicode=True,charset='utf8') 
    # 獲取游標(biāo) 
    self.cursor = self.connect.cursor() 
    print("connecting mysql success!") 
    self.answer = ['A','B','C','D']
  def process_item(self, item, spider):
    content = pymysql.escape_string(item['content'])
　　　　 # 獲取題目hash值，使用該字段過濾重復(fù)的題目
    m1 = md5.new()  
    m1.update(content)
    hash = m1.hexdigest()
    selectstr = "select id from question where hash='%s'"%(hash)
    self.cursor.execute(selectstr)
    res = self.cursor.fetchone()
    # 過濾相同的題目
    if not res:
　　　　　　　# 插入題目
      sqlstr = "insert into question(content,source,subject,level,answer,analysis,hash,answer_url) VALUES('%s','%s','%s','%s','%s','%s','%s','%s')"%(content,pymysql.escape_string(item['source']),item['subject'],item['level'],item['answer'],pymysql.escape_string(item['analysis']),hash,item['answer_url'])
      self.cursor.execute(sqlstr)
      qid = self.cursor.lastrowid
　　　　　　　# 插入選項(xiàng)
      for index in range(len(item['options'])):
        option = item['options'][index]
        answer = self.answer.index(item['answer'])
        if answer==index:
          ans = '2'
        else:
          ans = '1'
        sqlstr = "insert into options(content,qid,answer) VALUES('%s','%s','%s')"%(pymysql.escape_string(option[0]),qid,ans)
        self.cursor.execute(sqlstr)
      self.connect.commit() 
      #self.connect.close() 
    return item
 

scrapy crawl xueersi # scrapy crawl 爬蟲的名稱