# 1.spider文件

import scrapy
from wangyiPro.items import WangyiproItem
from selenium import webdriver

class WangyiSpider(scrapy.Spider):
  name = 'wangyi'
  # allowed_domains = ['www.xxx.con']
  start_urls = ['https://news.163.com/']
  # 瀏覽器實例化的操作只會被執(zhí)行一次
  bro = webdriver.Chrome(executable_path='chromedriver.exe')

  urls = []# 最終存放的就是5個板塊對應(yīng)的url

  def parse(self, response):
    li_list = response.xpath('//*[@id="index2016_wrap"]/div[1]/div[2]/div[2]/div[2]/div[2]/div/ul/li')
    for index in [3,4,6,7,8]:
      li = li_list[index]
      new_url = li.xpath('./a/@herf').extract_first()
      self.urls.append(new_url)

      # 對5大板塊對應(yīng)的url進行請求發(fā)送
      yield scrapy.Request(url=new_url,callback=self.parse_news)

  # 用來解析每一個板塊對應(yīng)的新聞數(shù)據(jù)【只能解析到新聞的標題】
  def parse_news(self,response):
    div_list = response.xpath('//div[@class="ndi_main"]/div')
    for div in div_list:
      title = div.xpath('./div/div[1]/h3/a/text()').extract_first()
      news_detail_url = div.xpath('./div/div[1]/h3/a/@href').extract_first()
      # 實例化item對象，將解析到的標題和內(nèi)容存儲到item對象中
      item = WangyiproItem()
      item['title'] = title
      # 對詳情頁的url進行手動請求發(fā)送獲得新聞內(nèi)容
      yield scrapy.Request(url=news_detail_url,callback=self.parse_detail,meta={'item':item})

  def parse_detail(self,response):
    item = response.meta['item']
    # 通過response解析出新聞內(nèi)容
    content = response.xpath('//div[@id="endText"]//text()').extract()
    content = ''.join(content)

    item['content'] = content
    yield item

  def close(self,spider):
    # 當爬蟲結(jié)束之后，調(diào)用關(guān)閉瀏覽器方法
    print('爬蟲整體結(jié)束~~~~~~~~~~~~~~~~~~~')
    self.bro.quit()
----------------------------------------------------------------------------------------
# 2.items文件

import scrapy

class WangyiproItem(scrapy.Item):
  # define the fields for your item here like:
  # name = scrapy.Field()
  title = scrapy.Field()
  content = scrapy.Field()
----------------------------------------------------------------------------------------
# 3.middlewares文件

from scrapy import signals
from scrapy.http import HtmlResponse
from time import sleep

class WangyiproDownloaderMiddleware(object):

  def process_request(self, request, spider):
    return None

  def process_response(self, request, response, spider):
    # 判斷哪些響應(yīng)對象是5個板塊的，如果在就對響應(yīng)對象進行處理
    if response.url in spider.urls:
      # 獲取在爬蟲類中定義好的瀏覽器
      bro = spider.bro
      bro.get(response.url)

      bro.execute_script('window.scrollTo(0,document.body.scrollHeight)')
      sleep(1)
      bro.execute_script('window.scrollTo(0,document.body.scrollHeight)')
      sleep(1)
      bro.execute_script('window.scrollTo(0,document.body.scrollHeight)')
      sleep(1)
      bro.execute_script('window.scrollTo(0,document.body.scrollHeight)')
      sleep(1)

      # 獲取攜帶了新聞數(shù)據(jù)的頁面源碼數(shù)據(jù)
      page_text = bro.page_source
      # 實例化一個新的響應(yīng)對象
      new_response = HtmlResponse(url=response.url,body=page_text,encoding='utf-8',request=request)
      return new_response
    else:
      return response

  def process_exception(self, request, exception, spider):
    pass
----------------------------------------------------------------------------------------
# 4.pipelines文件

class WangyiproPipeline(object):
  def process_item(self, item, spider):
    print(item)
    return item
----------------------------------------------------------------------------------------
# 5.setting文件
BOT_NAME = 'wangyiPro'

SPIDER_MODULES = ['wangyiPro.spiders']
NEWSPIDER_MODULE = 'wangyiPro.spiders'

USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36'

ROBOTSTXT_OBEY = False

DOWNLOADER_MIDDLEWARES = {
  'wangyiPro.middlewares.WangyiproDownloaderMiddleware': 543,
}

ITEM_PIPELINES = {
  'wangyiPro.pipelines.WangyiproPipeline': 300,
}

LOG_LEVEL = 'ERROR'