# -*- coding: utf-8 -*-
import scrapy
# 導(dǎo)入CrawlSpider相關(guān)模塊
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule

# 表示該爬蟲程序是基于CrawlSpider類的
class CrawldemoSpider(CrawlSpider):
  name = 'crawlDemo'  #爬蟲文件名稱
  #allowed_domains = ['www.qiushibaike.com']
  start_urls = ['http://www.qiushibaike.com/']
  
  #連接提取器：會去起始url響應(yīng)回來的頁面中提取指定的url
  link = LinkExtractor(allow=r'/8hr/page/\d+')
  #rules元組中存放的是不同的規(guī)則解析器（封裝好了某種解析規(guī)則)
  rules = (
    #規(guī)則解析器：可以將連接提取器提取到的所有連接表示的頁面進行指定規(guī)則（回調(diào)函數(shù)）的解析
    Rule(link, callback='parse_item', follow=True),
  )
  # 解析方法
  def parse_item(self, response):
    #print(response.url)
    divs = response.xpath('//div[@id="content-left"]/div')
    for div in divs:
      author = div.xpath('./div[@class="author clearfix"]/a[2]/h2/text()').extract_first()
      print(author)

# -*- coding: utf-8 -*-
import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from qiubaiBycrawl.items import QiubaibycrawlItem
import re
class QiubaitestSpider(CrawlSpider):
  name = 'qiubaiTest'
  #起始url
  start_urls = ['http://www.qiushibaike.com/']

  #定義鏈接提取器，且指定其提取規(guī)則
  page_link = LinkExtractor(allow=r'/8hr/page/\d+/')
  
  rules = (
    #定義規(guī)則解析器，且指定解析規(guī)則通過callback回調(diào)函數(shù)
    Rule(page_link, callback='parse_item', follow=True),
  )

  #自定義規(guī)則解析器的解析規(guī)則函數(shù)
  def parse_item(self, response):
    div_list = response.xpath('//div[@id="content-left"]/div')
    
    for div in div_list:
      #定義item
      item = QiubaibycrawlItem()
      #根據(jù)xpath表達式提取糗百中段子的作者
      item['author'] = div.xpath('./div/a[2]/h2/text()').extract_first().strip('\n')
      #根據(jù)xpath表達式提取糗百中段子的內(nèi)容
      item['content'] = div.xpath('.//div[@class="content"]/span/text()').extract_first().strip('\n')

      yield item #將item提交至管道

# -*- coding: utf-8 -*-

# Define here the models for your scraped items
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/items.html

import scrapy


class QiubaibycrawlItem(scrapy.Item):
  # define the fields for your item here like:
  # name = scrapy.Field()
  author = scrapy.Field() #作者
  content = scrapy.Field() #內(nèi)容

# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html

class QiubaibycrawlPipeline(object):
  
  def __init__(self):
    self.fp = None
    
  def open_spider(self,spider):
    print('開始爬蟲')
    self.fp = open('./data.txt','w')
    
  def process_item(self, item, spider):
    #將爬蟲文件提交的item寫入文件進行持久化存儲
    self.fp.write(item['author']+':'+item['content']+'\n')
    return item
  
  def close_spider(self,spider):
    print('結(jié)束爬蟲')
    self.fp.close()