Python爬蟲使用實例wallpaper問題記錄
1/ 排雷避坑
?? 中文亂碼問題
print(requests.get(url=url,headers=headers).text)

出現(xiàn)中文亂碼
原因分析:
<meta charset="gbk" />
解決方法:
法一:
response = requests.get(url=url,headers=headers) response.encoding = response.apparent_encoding # 自動轉碼, 防止中文亂碼 print(response.text)
法二:
print(requests.get(url=url,headers=headers).content.decode('gbk'))2/ 數(shù)據(jù)來源
css解析

for li in lis:
href = li.css('a::attr(href)').get()
title = li.css('b::text').get()
print(href, title)

刪掉標題為空的那一張圖

獲取圖片url
有的網(wǎng)站,保存的數(shù)據(jù)是裂開的圖片,可能是因為這個參數(shù):

3/ 正則處理
處理圖片url和標題的時候用了re模塊
電腦壁紙
通過匹配非數(shù)字字符并在遇到數(shù)字時截斷字符串
title1 = selector1.css('.photo .photo-pic img::attr(title)').get()
modified_title = re.split(r'\d', title1, 1)[0].strip()re.split(r'\d', title, 1)將 title 字符串按第一個數(shù)字進行分割。返回的列表的第一個元素就是數(shù)字前面的部分。strip() 去掉字符串首尾的空白字符。
url圖片路徑替換,因為從點開圖片到達的那個頁面無法得到的圖片路徑還是html頁面,不是https://····.jpg,所以替換成另一個可以獲取到的頁面。
https://sj.zol.com.cn/bizhi/detail_{num1}_{num2}.html正則替換修改為
https://app.zol.com.cn/bizhi/detail_{num1}.html例如 https://sj.zol.com.cn/bizhi/detail_12901_139948.html 轉換為 https://app.zol.com.cn/bizhi/detail_12901.html .
# https://sj.zol.com.cn/bizhi/detail_12901_139948.html url = "https://sj.zol.com.cn/bizhi/detail_12901_139948.html" pattern = r'https://sj\.zol\.com\.cn/bizhi/detail_(\d+)_\d+\.html' replacement = r'https://app.zol.com.cn/bizhi/detail_\1.html' new_url = re.sub(pattern, replacement, url) print(url,new_url)
4/ 電腦壁紙
?? 單線程單頁
適用于當頁面和第一頁
# python單線程爬取高清4k壁紙圖片
import os
import re
import requests
import parsel
url = 'https://pic.netbian.com/4kmeinv/' # 請求地址
# 模擬偽裝
headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.139 Safari/537.36'}
# response = requests.get(url=url,headers=headers)
# response.encoding = response.apparent_encoding # 自動轉碼, 防止中文亂碼
# print(response.text)
html_data = requests.get(url=url,headers=headers).content.decode('gbk')
# print(html_data)
selector = parsel.Selector(html_data)
lis = selector.css('.slist li')
for li in lis:
#href = li.css('a::attr(href)').get()
title = li.css('b::text').get()
if title:
+ li.css('a::attr(href)').get()
response = requests.get(url=href, headers=headers)
#print(href, title)
# 這里只是獲取頁面
# img_content = requests.get(url=href, headers=headers).content
# 不可行, 都是同一張圖 https://pic.netbian.com/uploads/allimg/230813/221347-16919360273e05.jpg
+ li.css('a::attr(href)').get()
response1 = requests.get(url=href, headers=headers).content.decode('gbk')
selector1 = parsel.Selector(response1)
# 若要標題亂碼,此處可不解碼
# response1 = requests.get(url=href, headers=headers)
# selector1 = parsel.Selector(response1.text)
# img_url = selector1.css('.slist li img::attr(src)').get()
# 這一步錯了, 要去href頁面找img_url, 這是在原來的url頁面找了
img_url = 'https://pic.netbian.com' + selector1.css('.photo .photo-pic img::attr(src)').get()
img_content = requests.get(url=img_url,headers=headers).content
# 順便更新一下title, 因為原來的是半截的, 不全
title1 = selector1.css('.photo .photo-pic img::attr(title)').get()
modified_title = re.split(r'\d', title1, 1)[0].strip()
with open('img\\'+modified_title+'.jpg',mode='wb') as f:
f.write(img_content)
#print(href, title)
print('正在保存:', modified_title, img_url)?? 單線程多page
適用于從第二頁開始的多頁
# python單線程爬取高清4k壁紙圖片
import os
import re
import time
import requests
import parsel
# url的規(guī)律
# https://pic.netbian.com/new/index.html
# https://pic.netbian.com/new/index_1.html
# https://pic.netbian.com/new/index_2.html
# ...
start_time = time.time()
for page in range(2,10):
print(f'--------- 正在爬取第{page}的內(nèi)容 ----------')
url = f'https://pic.netbian.com/4kmeinv/index_{page}.html' # 請求地址
# 模擬偽裝
headers = {
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.139 Safari/537.36'}
# response = requests.get(url=url,headers=headers)
# response.encoding = response.apparent_encoding # 自動轉碼, 防止中文亂碼
# print(response.text)
html_data = requests.get(url=url, headers=headers).content.decode('gbk')
# print(html_data)
selector = parsel.Selector(html_data)
lis = selector.css('.slist li')
for li in lis:
# href = li.css('a::attr(href)').get()
title = li.css('b::text').get()
if title:
+ li.css('a::attr(href)').get()
response = requests.get(url=href, headers=headers)
# print(href, title)
# 這里只是獲取頁面
# img_content = requests.get(url=href, headers=headers).content
# 不可行, 都是同一張圖 https://pic.netbian.com/uploads/allimg/230813/221347-16919360273e05.jpg
+ li.css('a::attr(href)').get()
response1 = requests.get(url=href, headers=headers).content.decode('gbk')
selector1 = parsel.Selector(response1)
# 若要標題亂碼,此處可不解碼
# response1 = requests.get(url=href, headers=headers)
# selector1 = parsel.Selector(response1.text)
# img_url = selector1.css('.slist li img::attr(src)').get()
# 這一步錯了, 要去href頁面找img_url, 這是在原來的url頁面找了
img_url = 'https://pic.netbian.com' + selector1.css('.photo .photo-pic img::attr(src)').get()
img_content = requests.get(url=img_url, headers=headers).content
# 順便更新一下title, 因為原來的是半截的, 不全
title1 = selector1.css('.photo .photo-pic img::attr(title)').get()
modified_title = re.split(r'\d', title1, 1)[0].strip()
with open('img\\' + modified_title + '.jpg', mode='wb') as f:
f.write(img_content)
# print(href, title)
print('正在保存:', modified_title, img_url)
stop_time = time.time()
print(f'耗時:{int(stop_time)-int(start_time)}秒')運行效果:


?? 多線程多頁
# python多線程爬取高清4k壁紙圖片
import os
import re
import time
import requests
import parsel
import concurrent.futures
def get_img(url):
# 模擬偽裝
headers = {
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.139 Safari/537.36'}
# response = requests.get(url=url,headers=headers)
# response.encoding = response.apparent_encoding # 自動轉碼, 防止中文亂碼
# print(response.text)
html_data = requests.get(url=url, headers=headers).content.decode('gbk')
# print(html_data)
selector = parsel.Selector(html_data)
lis = selector.css('.slist li')
for li in lis:
# href = li.css('a::attr(href)').get()
title = li.css('b::text').get()
if title:
+ li.css('a::attr(href)').get()
response = requests.get(url=href, headers=headers)
# print(href, title)
# 這里只是獲取頁面
# img_content = requests.get(url=href, headers=headers).content
# 不可行, 都是同一張圖 https://pic.netbian.com/uploads/allimg/230813/221347-16919360273e05.jpg
+ li.css('a::attr(href)').get()
response1 = requests.get(url=href, headers=headers).content.decode('gbk')
selector1 = parsel.Selector(response1)
# 若要標題亂碼,此處可不解碼
# response1 = requests.get(url=href, headers=headers)
# selector1 = parsel.Selector(response1.text)
# img_url = selector1.css('.slist li img::attr(src)').get()
# 這一步錯了, 要去href頁面找img_url, 這是在原來的url頁面找了
img_url = 'https://pic.netbian.com' + selector1.css('.photo .photo-pic img::attr(src)').get()
img_content = requests.get(url=img_url, headers=headers).content
# 順便更新一下title, 因為原來的是半截的, 不全
title1 = selector1.css('.photo .photo-pic img::attr(title)').get()
modified_title = re.split(r'\d', title1, 1)[0].strip()
img_folder = 'img1\\'
if not os.path.exists(img_folder):
os.makedirs(img_folder)
with open(img_folder + modified_title + '.jpg', mode='wb') as f:
f.write(img_content)
# print(href, title)
print('正在保存:', modified_title, img_url)
def main(url):
get_img(url)
start_time = time.time()
executor = concurrent.futures.ThreadPoolExecutor(max_workers=5)
for page in range(2, 12):
print(f'--------- 正在爬取第{page}的內(nèi)容 ----------')
url = f'https://pic.netbian.com/4kmeinv/index_{page}.html' # 請求地址
executor.submit(main, url)
executor.shutdown()
stop_time = time.time()
print(f'耗時:{int(stop_time) - int(start_time)}秒')5/ 手機壁紙
類似地,另一個網(wǎng)站,圖片集合多頁,點開之后里面有多張圖片
先試圖獲取外部的,再獲取里面的,然后2個一起
?? 單線程單頁0
import os
import re
import requests
import parsel
url = 'https://sj.zol.com.cn/bizhi/5/' # 請求地址
# 模擬偽裝
headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.139 Safari/537.36'}
# response = requests.get(url=url,headers=headers)
# response.encoding = response.apparent_encoding # 自動轉碼, 防止中文亂碼
# print(response.text)
response = requests.get(url=url,headers=headers)
#print(response.text)
selector = parsel.Selector(response.text)
lis = selector.css('.pic-list2 li')
#img_name=1
for li in lis:
#href = li.css('a::attr(href)').get()
title = li.css('.pic img::attr(title)').get()
#href = li.css('.pic img::attr(src)').get()
#print(title, href)
if title:
# +li.css('a::attr(href)').get()
# https://sj.zol.com.cn/bizhi/detail_12901_139948.html
# https://app.zol.com.cn/bizhi/detail_12901_139948.html#p1
# + li.css('a::attr(href)').get() + '#p1'
href=li.css('img::attr(src)').get()
#print(href, title)
# + li.css('a::attr(href)').get() + '#p1'
#response1 = requests.get(url=href, headers=headers).content.decode('utf-8')
#selector1 = parsel.Selector(response1)
#img_url=selector1.css('.gallery li img::attr(src)').get()
#print(img_url)
# 這里只是獲取頁面
img_content = requests.get(url=href, headers=headers).content
# 不可行, 都是同一張圖 https://pic.netbian.com/uploads/allimg/230813/221347-16919360273e05.jpg
# https://sj.zol.com.cn/bizhi/detail_12901_139948.html
# https://app.zol.com.cn/bizhi/detail_12901_139948.html#p1
#href= selector1.css('.photo-list-box li::attr(href)').get()
# + + '#p1'
#response2 = requests.get(url=href, headers=headers)
#selector2 = parsel.Selector(response2.text)
#print(href)
# 若要標題亂碼,此處可不解碼
# response1 = requests.get(url=href, headers=headers)
# selector1 = parsel.Selector(response1.text)
# img_url = selector1.css('.slist li img::attr(src)').get()
# 這一步錯了, 要去href頁面找img_url, 這是在原來的url頁面找了
#img_url = selector1.css('.gallery img::attr(src)').get()
#img_content = requests.get(url=img_url, headers=headers).content
#print(img_url)
# 順便更新一下title, 因為原來的是半截的, 不全
# title1 = selector1.css('.photo .photo-pic img::attr(title)').get()
img_folder = 'img3\\'
if not os.path.exists(img_folder):
os.makedirs(img_folder)
with open(img_folder + title + '.jpg', mode='wb') as f:
f.write(img_content)
# print(href, title)
print('正在保存:', title, href)
#img_name += 1?? 單線程單頁1
# 下載子頁面全部
import os
import requests
import parsel
url = 'https://app.zol.com.cn/bizhi/detail_12901.html' # 請求地址
# 模擬偽裝
headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.139 Safari/537.36'}
response = requests.get(url=url,headers=headers)
selector = parsel.Selector(response.text)
lis = selector.css('.album-list li')
i = 0
for li in lis:
# Get all img elements within the current li
img_tags = li.css('img::attr(src)').getall() # This gets all the img src attributes
for href in img_tags: # Iterate over all img src attributes
img_content = requests.get(url=href, headers=headers).content
img_folder = 'img4\\'
if not os.path.exists(img_folder):
os.makedirs(img_folder)
with open(img_folder + str(i) + '.jpg', mode='wb') as f:
f.write(img_content)
# print(href, i)
print('正在保存:', i, href)
i += 1 # Increment i for each image saved?? 單線程單頁
import os
import re
import requests
import parsel
url = 'https://sj.zol.com.cn/bizhi/5/' # 請求地址
# 模擬偽裝
headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.139 Safari/537.36'}
response = requests.get(url=url,headers=headers)
#print(response.text)
selector = parsel.Selector(response.text)
#lis = selector.css('.pic-list2 li')
# 篩除包含的底部 3個 猜你喜歡
lis=selector.css('.pic-list2 .photo-list-padding')
for li in lis:
#href = li.css('a::attr(href)').get()
title = li.css('.pic img::attr(title)').get()
href = li.css('a::attr(href)').get()
#print(title, href)
# https://sj.zol.com.cn/bizhi/detail_12901_139948.html
#url = "https://sj.zol.com.cn/bizhi/detail_12901_139948.html"
pattern = r'/bizhi/detail_(\d+)_\d+\.html'
replacement = r'https://app.zol.com.cn/bizhi/detail_\1.html'
new_url = re.sub(pattern, replacement, href)
#print(href, new_url)
#url = 'https://app.zol.com.cn/bizhi/detail_12901.html' # 請求地址
# 模擬偽裝
headers = {
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.139 Safari/537.36'}
response = requests.get(url=new_url, headers=headers)
selector = parsel.Selector(response.text)
lis1 = selector.css('.album-list li')
i = 0
for li1 in lis1:
# Get all img elements within the current li
img_tags = li1.css('img::attr(src)').getall() # This gets all the img src attributes
for href in img_tags: # Iterate over all img src attributes
img_content = requests.get(url=href, headers=headers).content
img_folder = 'img5\\'
if not os.path.exists(img_folder):
os.makedirs(img_folder)
with open(img_folder + title+'_'+str(i) + '.jpg', mode='wb') as f:
f.write(img_content)
# print(href, i)
print('正在保存:',title+'_'+str(i), href)
i += 1 # Increment i for each image saved
?? 單線程多頁
import os
import re
import requests
import parsel
for page in range(1,3):
print(f'--------- 正在爬取第{page}的內(nèi)容 ----------')
if page==1:
url = 'https://sj.zol.com.cn/bizhi/5/' # 請求地址
else:
url = f'https://sj.zol.com.cn/bizhi/5/{page}.html' # 請求地址
# 模擬偽裝
headers = {
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.139 Safari/537.36'}
response = requests.get(url=url, headers=headers)
# print(response.text)
selector = parsel.Selector(response.text)
# lis = selector.css('.pic-list2 li')
# 篩除包含的底部 3個 猜你喜歡
lis = selector.css('.pic-list2 .photo-list-padding')
for li in lis:
# href = li.css('a::attr(href)').get()
title = li.css('.pic img::attr(title)').get()
href = li.css('a::attr(href)').get()
# print(title, href)
# https://sj.zol.com.cn/bizhi/detail_12901_139948.html
# url = "https://sj.zol.com.cn/bizhi/detail_12901_139948.html"
pattern = r'/bizhi/detail_(\d+)_\d+\.html'
replacement = r'https://app.zol.com.cn/bizhi/detail_\1.html'
new_url = re.sub(pattern, replacement, href)
# print(href, new_url)
# url = 'https://app.zol.com.cn/bizhi/detail_12901.html' # 請求地址
# 模擬偽裝
headers = {
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.139 Safari/537.36'}
response = requests.get(url=new_url, headers=headers)
selector = parsel.Selector(response.text)
lis1 = selector.css('.album-list li')
i = 0
for li1 in lis1:
# Get all img elements within the current li
img_tags = li1.css('img::attr(src)').getall() # This gets all the img src attributes
for href in img_tags: # Iterate over all img src attributes
img_content = requests.get(url=href, headers=headers).content
img_folder = 'img6\\'
if not os.path.exists(img_folder):
os.makedirs(img_folder)
with open(img_folder + title + '_' + str(i) + '.jpg', mode='wb') as f:
f.write(img_content)
# print(href, i)
print('正在保存:', title + '_' + str(i), href)
i += 1 # Increment i for each image saved?? 多線程多頁
import os
import re
import time
import requests
import parsel
import concurrent.futures
def get_imgs(url):
# 模擬偽裝
headers = {
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.139 Safari/537.36'}
response = requests.get(url=url, headers=headers)
# print(response.text)
selector = parsel.Selector(response.text)
# lis = selector.css('.pic-list2 li')
# 篩除包含的底部 3個 猜你喜歡
lis = selector.css('.pic-list2 .photo-list-padding')
for li in lis:
# href = li.css('a::attr(href)').get()
title = li.css('.pic img::attr(title)').get()
href = li.css('a::attr(href)').get()
# print(title, href)
# https://sj.zol.com.cn/bizhi/detail_12901_139948.html
# url = "https://sj.zol.com.cn/bizhi/detail_12901_139948.html"
pattern = r'/bizhi/detail_(\d+)_\d+\.html'
replacement = r'https://app.zol.com.cn/bizhi/detail_\1.html'
new_url = re.sub(pattern, replacement, href)
# print(href, new_url)
# url = 'https://app.zol.com.cn/bizhi/detail_12901.html' # 請求地址
# 模擬偽裝
headers = {
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.139 Safari/537.36'}
response = requests.get(url=new_url, headers=headers)
selector = parsel.Selector(response.text)
lis1 = selector.css('.album-list li')
i = 0
for li1 in lis1:
# Get all img elements within the current li
img_tags = li1.css('img::attr(src)').getall() # This gets all the img src attributes
for href in img_tags: # Iterate over all img src attributes
img_content = requests.get(url=href, headers=headers).content
img_folder = 'img7\\'
if not os.path.exists(img_folder):
os.makedirs(img_folder)
with open(img_folder + title + '_' + str(i) + '.jpg', mode='wb') as f:
f.write(img_content)
# print(href, i)
print('正在保存:', title + '_' + str(i), href)
i += 1 # Increment i for each image saved
def main(url):
get_imgs(url)
start_time = time.time()
executor = concurrent.futures.ThreadPoolExecutor(max_workers=4)
for page in range(1, 9):
#print(f'--------- 正在爬取第{page}的內(nèi)容 ----------')
if page == 1:
url = 'https://sj.zol.com.cn/bizhi/5/' # 請求地址
else:
url = f'https://sj.zol.com.cn/bizhi/5/{page}.html' # 請求地址
executor.submit(main, url)
executor.shutdown()
stop_time = time.time()
print(f'耗時:{int(stop_time) - int(start_time)}秒')到此這篇關于Python爬蟲使用實例-wallpaper的文章就介紹到這了,更多相關Python wallpaper內(nèi)容請搜索腳本之家以前的文章或繼續(xù)瀏覽下面的相關文章希望大家以后多多支持腳本之家!
相關文章
基于進程內(nèi)通訊的python聊天室實現(xiàn)方法
這篇文章主要介紹了基于進程內(nèi)通訊的python聊天室實現(xiàn)方法,實例分析了Python聊天室的相關實現(xiàn)技巧,需要的朋友可以參考下2015-06-06
Python使用PyQt5/PySide2編寫一個極簡的音樂播放器功能
這篇文章主要介紹了Python中使用PyQt5/PySide2編寫一個極簡的音樂播放器功能,本文通過實例代碼給大家介紹的非常詳細,具有一定的參考借鑒價值,需要的朋友可以參考下2020-02-02
pycharm遠程調(diào)試openstack的圖文教程
這篇文章主要為大家詳細介紹了pycharm遠程調(diào)試openstack的圖文教程,具有一定的參考價值,感興趣的小伙伴們可以參考一下2017-11-11
使用Python實現(xiàn)MongoDB數(shù)據(jù)轉表格文件CSV
這篇文章主要為大家詳細介紹了如何使用Python實現(xiàn)將MongoDB中的數(shù)據(jù)轉換為表格文件(如CSV)以便于數(shù)據(jù)交換、共享或導入到其他系統(tǒng)進行分析,需要的可以參考下2024-04-04
使用Python、TensorFlow和Keras來進行垃圾分類的操作方法
這篇文章主要介紹了如何使用Python、TensorFlow和Keras來進行垃圾分類,這個模型在測試集上可以達到約80%的準確率,可以作為一個基礎模型進行后續(xù)的優(yōu)化,需要的朋友可以參考下2023-05-05
使用BeautifulSoup爬蟲程序獲取百度搜索結果的標題和url示例
這篇文章主要介紹了使用BeautifulSoup編寫了一段爬蟲程序獲取百度搜索結果的標題和url的示例,大家參考使用吧2014-01-01
Python多進程池 multiprocessing Pool用法示例
這篇文章主要介紹了Python多進程池 multiprocessing Pool用法,結合實例形式分析了多進程池 multiprocessing Pool相關概念、原理及簡單使用技巧,需要的朋友可以參考下2018-09-09

