pip install Scrapy

pip --version

pip install Scrapy

H:\python\scrapyDemo>scrapy startproject tutorial
New Scrapy project 'tutorial', using template directory 'f:\\python27\\lib\\site-packages\\scrapy\\templates\\project', created in:
  H:\python\scrapyDemo\tutorial

You can start your first spider with:
  cd tutorial
  scrapy genspider example example.com

# -*- coding: utf-8 -*-
# Define here the models for your scraped items
#
# See documentation in:
# http://doc.scrapy.org/en/latest/topics/items.html
import scrapy
from scrapy.item import Item, Field
class TutorialItem(Item):
  title = Field()
  author = Field()
  releasedate = Field()

# -*-coding:utf-8-*-
import sys
from scrapy.linkextractors.sgml import SgmlLinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from tutorial.items import TutorialItem
reload(sys)
sys.setdefaultencoding("utf-8")
class ListSpider(CrawlSpider):
  # 爬蟲(chóng)名稱(chēng)
  name = "tutorial"
  # 設(shè)置下載延時(shí)
  download_delay = 1
  # 允許域名
  allowed_domains = ["news.cnblogs.com"]
  # 開(kāi)始URL
  start_urls = [
    "https://news.cnblogs.com"
  ]
  # 爬取規(guī)則,不帶callback表示向該類(lèi)url遞歸爬取
  rules = (
    Rule(SgmlLinkExtractor(allow=(r'https://news.cnblogs.com/n/page/\d',))),
    Rule(SgmlLinkExtractor(allow=(r'https://news.cnblogs.com/n/\d+',)), callback='parse_content'),
  )

  # 解析內(nèi)容函數(shù)
  def parse_content(self, response):
    item = TutorialItem()

    # 當(dāng)前URL
    title = response.selector.xpath('//div[@id="news_title"]')[0].extract().decode('utf-8')
    item['title'] = title

    author = response.selector.xpath('//div[@id="news_info"]/span/a/text()')[0].extract().decode('utf-8')
    item['author'] = author

    releasedate = response.selector.xpath('//div[@id="news_info"]/span[@class="time"]/text()')[0].extract().decode(
      'utf-8')
    item['releasedate'] = releasedate

    yield item

# -*- coding: utf-8 -*-
# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html
import json
import codecs
class TutorialPipeline(object):
  def __init__(self):
    self.file = codecs.open('data.json', mode='wb', encoding='utf-8')#數(shù)據(jù)存儲(chǔ)到data.json

  def process_item(self, item, spider):
    line = json.dumps(dict(item)) + "\n"
    self.file.write(line.decode("unicode_escape"))

    return item

# -*- coding: utf-8 -*-
BOT_NAME = 'tutorial'
SPIDER_MODULES = ['tutorial.spiders']
NEWSPIDER_MODULE = 'tutorial.spiders'

# 禁止cookies,防止被ban
COOKIES_ENABLED = False
COOKIES_ENABLES = False

# 設(shè)置Pipeline,此處實(shí)現(xiàn)數(shù)據(jù)寫(xiě)入文件
ITEM_PIPELINES = {
  'tutorial.pipelines.TutorialPipeline': 300
}

# 設(shè)置爬蟲(chóng)爬取的最大深度
DEPTH_LIMIT = 100

from scrapy import cmdline
cmdline.execute("scrapy crawl tutorial".split())