#coding=utf-8 
#北京及周邊省會(huì)城市污染數(shù)據(jù)、天氣數(shù)據(jù)每小時(shí)監(jiān)測(cè)值爬蟲(chóng)程序 
import urllib.request 
import re 
import urllib.error 
import time 
#模擬成瀏覽器 
headers=("User-Agent","Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36") 
opener = urllib.request.build_opener() 
opener.addheaders=[headers] 
#將opener安裝為全局 
urllib.request.install_opener(opener) 
def get_pm25_and_weather(city): 
 #首先執(zhí)行獲取空氣質(zhì)量數(shù)據(jù)，返回?cái)?shù)據(jù)更新時(shí)間 
 data_time=getpm25(city) 
 #然后將獲取到的數(shù)據(jù)更新時(shí)間賦值給獲取天氣數(shù)據(jù)函數(shù)使用 
 getweather(city,data_time) 
def getpm25(city): 
 try: 
 #設(shè)置url地址 
 url="http://pm25.in/"+city 
 data=urllib.request.urlopen(url).read().decode("utf-8") 
 print("城市："+city) 
 #構(gòu)建數(shù)據(jù)更新時(shí)間的表達(dá)式 
 data_time='<div class="live_data_time">\s{1,}<p>數(shù)據(jù)更新時(shí)間：(.*?)</p>' 
 #尋找出數(shù)據(jù)更新時(shí)間 
 datatime=re.compile(data_time, re.S).findall(data) 
 print("數(shù)據(jù)更新時(shí)間："+datatime[0]) 
 #構(gòu)建數(shù)據(jù)收集的表達(dá)式 
 data_pm25 = '<div class="span1">\s{1,}<div class="value">\n\s{1,}(.*?)\s{1,}</div>' 
 data_o3='<div class="span1">\s{1,}<div class ="value">\n\s{1,}(.*?)\s{1,}</div>' 
 #尋找出所有的監(jiān)測(cè)值 
 pm25list = re.compile(data_pm25, re.S).findall(data) 
 o3list=re.compile(data_o3, re.S).findall(data) 
 #將臭氧每小時(shí)的值插入到原列表中 
 pm25list.append(o3list[0]) 
 print("AQI指數(shù)，PM2.5，PM10，CO，NO2，SO2，O3：（單位：μg/m3，CO為mg/m3）") 
 print(pm25list) 
 #將獲取到的值寫(xiě)入文件中 
 writefiles_pm25(city,datatime,pm25list) 
 #返回?cái)?shù)據(jù)更新時(shí)間值 
 return datatime 
 except urllib.error.URLError as e: 
 print("出現(xiàn)URLERROR！一分鐘后重試……") 
 if hasattr(e,"code"): 
  print(e.code) 
 if hasattr(e,"reason"): 
  print(e.reason) 
 time.sleep(60) 
 #出現(xiàn)異常則過(guò)一段時(shí)間重新執(zhí)行此部分 
 getpm25(city) 
 except Exception as e: 
 print("出現(xiàn)EXCEPTION！十秒鐘后重試……") 
 print("Exception："+str(e)) 
 time.sleep(10) 
 # 出現(xiàn)異常則過(guò)一段時(shí)間重新執(zhí)行此部分 
 getpm25(city) 
def writefiles_pm25(filename,datatime,pm25list): 
 #將獲取的數(shù)據(jù)寫(xiě)入文件中，數(shù)據(jù)分別為時(shí)間，AQI指數(shù)，PM2.5，PM10，CO，NO2，SO2，O3。（單位：μg/m3，CO為mg/m3） 
 f = open("D:\Python\Python35\myweb\data_pm25\data_pm25_"+filename+".txt", "a") 
 f.write(datatime[0]) 
 f.write(",") 
 for pm25 in pm25list: 
 f.write(str(pm25)) 
 f.write(",") 
 f.write("\n") 
 print("該條空氣質(zhì)量數(shù)據(jù)已添加到文件中！") 
 f.close() 
def getweather(city,datatime): 
 try: 
 #構(gòu)建url 
 url="http://"+city+".tianqi.com/" 
 data=urllib.request.urlopen(url).read().decode("gbk") 
 #構(gòu)建數(shù)據(jù)收集的表達(dá)式 
 data_weather = '<li class="cDRed">(.*?)</li>' 
 data_wind='<li style="height:18px;overflow:hidden">(.*?)</li>' 
 data_temperature='<div id="rettemp"><strong>(.*?)°' 
 data_humidity='</strong><span>相對(duì)濕度：(.*?)</span>' 
 #尋找出所有的監(jiān)測(cè)值 
 weatherlist = re.compile(data_weather, re.S).findall(data) 
 windlist=re.compile(data_wind, re.S).findall(data) 
 temperaturelist = re.compile(data_temperature, re.S).findall(data) 
 humiditylist = re.compile(data_humidity, re.S).findall(data) 
 #將其他值插入到天氣列表中 
 weatherlist.append(windlist[0]) 
 weatherlist.append(temperaturelist[0]) 
 weatherlist.append(humiditylist[0]) 
 print("天氣狀況，風(fēng)向風(fēng)速，實(shí)時(shí)溫度，相對(duì)濕度：") 
 print(weatherlist) 
 #將獲取到的值寫(xiě)入文件中 
 writefiles_weather(city,datatime,weatherlist) 
 except urllib.error.URLError as e: 
 print("出現(xiàn)URLERROR！一分鐘后重試……") 
 if hasattr(e,"code"): 
  print(e.code) 
 if hasattr(e,"reason"): 
  print(e.reason) 
 time.sleep(60) 
 # 出現(xiàn)異常則過(guò)一段時(shí)間重新執(zhí)行此部分 
 getweather(city,datatime) 
 except Exception as e: 
 print("出現(xiàn)EXCEPTION！十秒鐘后重試……") 
 print("Exception："+str(e)) 
 time.sleep(10) 
 # 出現(xiàn)異常則過(guò)一段時(shí)間重新執(zhí)行此部分 
 getweather(city, datatime) 
def writefiles_weather(filename,datatime,weatherlist): 
 #將獲取的數(shù)據(jù)寫(xiě)入文件中，數(shù)據(jù)分別為時(shí)間，天氣狀況，風(fēng)向風(fēng)速，實(shí)時(shí)溫度，相對(duì)濕度。 
 f = open("D:\Python\Python35\myweb\data_weather\data_weather_"+filename+".txt", "a") 
 f.write(datatime[0]) 
 f.write(",") 
 for weather in weatherlist: 
 f.write(str(weather)) 
 f.write(",") 
 f.write("\n") 
 print("該條天氣數(shù)據(jù)已添加到文件中！") 
 f.close() 
#退出循環(huán)可用Ctrl+C鍵 
while True: 
 print("開(kāi)始工作！") 
 get_pm25_and_weather("beijing") 
 get_pm25_and_weather("tianjin") 
 get_pm25_and_weather("shijiazhuang") 
 get_pm25_and_weather("taiyuan") 
 get_pm25_and_weather("jinan") 
 get_pm25_and_weather("shenyang") 
 get_pm25_and_weather("huhehaote") 
 get_pm25_and_weather("zhengzhou") 
 #每一小時(shí)執(zhí)行一次 
 print("休息中……") 
 print("\n") 
 time.sleep(3600)