from bs4 import BeautifulSoup 
import requests
import time
# 分別導(dǎo)入time、requests、BeautifulSoup庫(kù)

url = 'http://www.365haoshu.com/Book/Chapter/'
# 鏈接地址url，這兒url章節(jié)鏈接沒(méi)全寫出來(lái)是因?yàn)橄旅娅@取章節(jié)鏈接時(shí)要用到這些url
req = requests.get(url+'List.aspx?NovelId=0326')
# 打開(kāi)章節(jié)頁(yè)面，
req_bf = BeautifulSoup(req.text,"html.parser")
print（req_bf）
# 將打開(kāi)的頁(yè)面以text打印出來(lái)
div = req_bf.find_all('div',class_='user-catalog-ul-li')
# 分析頁(yè)面，所需要的章節(jié)名和章節(jié)鏈接是在div標(biāo)簽，屬性class為user-catalog-ul-li下
# 找到這個(gè)下的內(nèi)容，并打印
s = []
for d in div:
  s.append(d.find('a'))
print(s)
# 獲取div下面的a標(biāo)簽下的內(nèi)容
names=[] # 存儲(chǔ)章節(jié)名
hrefs=[] # 存儲(chǔ)章節(jié)鏈接
for i in s:
  names.append(i.find('span').string)
  hrefs.append(url + i.get('href'))
# 將所有的章節(jié)和章節(jié)鏈接存入的列表中
觀察href后的鏈接和打開(kāi)章節(jié)內(nèi)容頁(yè)面的鏈接是不完全的相同的， 所以要拼接使得瀏覽器能直接打開(kāi)章節(jié)內(nèi)容

txt = requests.get(hrefs[0])
div_bf = BeautifulSoup(txt.text,'html.parser')
div = div_bf.find_all('div',class_='container user-reading-online pos-rel')
#print(div)
ps = BeautifulSoup(str(div),"html.parser")
p=ps.find_all('p',class_='p-content')
print(p)
txt=[]
for i in p:
  txt.append(i.string+'\n')
print(txt)

# --*-- coding=utf-8 --*--

from bs4 import BeautifulSoup
import requests
import time


class spiderstory(object):

  def __init__(self): # 初始化
    self.url = 'http://www.365haoshu.com/Book/Chapter/'
    self.names = [] # 存放章節(jié)名
    self.hrefs = [] # 存放章節(jié)鏈接

  def get_urlAndName(self):
    '''獲取章節(jié)名和章節(jié)鏈接'''
    req = requests.get(url=self.url+'List.aspx?NovelId=0326') # 獲取章節(jié)目錄頁(yè)面
    time.sleep(1) # 等待1秒
    div_bf = BeautifulSoup(req.text,"html.parser") # req后面跟text和html都行
    div = div_bf.find_all('div', class_='user-catalog-ul-li') # 查找內(nèi)容，標(biāo)簽為div，屬性為class='user-catalog-ul-li'
    a_bf = BeautifulSoup(str(div))
    a = a_bf.find_all('a') # # 查找內(nèi)容，標(biāo)簽為a
    for i in a:
      self.names.append(i.find('span').string) # 獲取內(nèi)容直接string就行
      self.hrefs.append(self.url + i.get('href')) # 獲取鏈接

  def get_text(self,url):
    '''獲取章節(jié)內(nèi)容'''
    req = requests.get(url=url)
    div_bf = BeautifulSoup(req.text,"html.parser")
    div = div_bf.find_all('div', class_='container user-reading-online pos-rel') # 查找內(nèi)容
    ps = BeautifulSoup(str(div), "html.parser")
    p = ps.find_all('p', class_='p-content')
    text = []
    for each in p:
      text.append(each.string)
    print(text)

    return text # 將獲得的內(nèi)容返回

  def writer(self, name, path, text):
    '''寫入text文檔中'''
    with open(path, 'a', encoding='utf-8') as f:
      f.write(name + '\n')
      f.writelines(text)
      f.write('\n\n')
if __name__ == "__main__": # 運(yùn)行入口
  s = spiderstory()
  s.get_urlAndName()
  le = len(s.names)
  for i in range(le): # 利用for循環(huán)獲得所有的內(nèi)容
    name = s.names[i]
    text = str(s.get_text(s.hrefs[i]))
    s.writer(name,"我以月夜寄相思.txt",text)
  print('下載完畢?。。?)