def read_page(url, page_num, keyword): # 模仿瀏覽器post需求信息，并讀取返回后的頁面信息
  page_headers = {
    'Host': 'www.lagou.com',
    'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) '
           'Chrome/45.0.2454.85 Safari/537.36 115Browser/6.0.3',
    'Connection': 'keep-alive'
    }
  if page_num == 1:
    boo = 'true'
  else:
    boo = 'false'
  page_data = parse.urlencode([  # 通過頁面分析，發(fā)現(xiàn)瀏覽器提交的FormData包括以下參數(shù)
    ('first', boo),
    ('pn', page_num),
    ('kd', keyword)
    ])
  req = request.Request(url, headers=page_headers)
  page = request.urlopen(req, data=page_data.encode('utf-8')).read()
  page = page.decode('utf-8')
  return page

def read_tag(page, tag):
  page_json = json.loads(page)
  page_json = page_json['content']['result'] 
  # 通過分析獲取的json信息可知，招聘信息包含在返回的result當(dāng)中，其中包含了許多其他參數(shù)
  page_result = [num for num in range(15)] # 構(gòu)造一個容量為15的占位list，用以構(gòu)造接下來的二維數(shù)組
  for i in range(15):
    page_result[i] = [] # 構(gòu)造二維數(shù)組
    for page_tag in tag:
      page_result[i].append(page_json[i].get(page_tag)) # 遍歷參數(shù)，將它們放置在同一個list當(dāng)中
    page_result[i][8] = ','.join(page_result[i][8])
  return page_result  # 返回當(dāng)前頁的招聘信息

def save_excel(fin_result, tag_name, file_name):
  book = Workbook(encoding='utf-8')
  tmp = book.add_sheet('sheet')
  times = len(fin_result)+1
  for i in range(times): # i代表的是行,i+1代表的是行首信息
    if i == 0:
      for tag_name_i in tag_name:
        tmp.write(i, tag_name.index(tag_name_i), tag_name_i)
    else:
      for tag_list in range(len(tag_name)):
        tmp.write(i, tag_list, str(fin_result[i-1][tag_list]))
  book.save(r'C:\Users\Administrator\Desktop\%s.xls' % file_name)

def save_excel(fin_result, tag_name, file_name): # 將抓取到的招聘信息存儲到excel當(dāng)中
  book = xlsxwriter.Workbook(r'C:\Users\Administrator\Desktop\%s.xls' % file_name) # 默認(rèn)存儲在桌面上
  tmp = book.add_worksheet()
  row_num = len(fin_result)
  for i in range(1, row_num):
    if i == 1:
      tag_pos = 'A%s' % i
      tmp.write_row(tag_pos, tag_name)
    else:
      con_pos = 'A%s' % i
      content = fin_result[i-1] # -1是因?yàn)楸槐砀竦谋眍^所占
      tmp.write_row(con_pos, content)
  book.close()

#! -*-coding:utf-8 -*-

from urllib import request, parse
from bs4 import BeautifulSoup as BS
import json
import datetime
import xlsxwriter

starttime = datetime.datetime.now()

url = r'http://www.lagou.com/jobs/positionAjax.json?city=%E5%8C%97%E4%BA%AC'
# 拉鉤網(wǎng)的招聘信息都是動態(tài)獲取的，所以需要通過post來遞交json信息，默認(rèn)城市為北京

tag = ['companyName', 'companyShortName', 'positionName', 'education', 'salary', 'financeStage', 'companySize',
    'industryField', 'companyLabelList'] # 這是需要抓取的標(biāo)簽信息，包括公司名稱，學(xué)歷要求，薪資等等

tag_name = ['公司名稱', '公司簡稱', '職位名稱', '所需學(xué)歷', '工資', '公司資質(zhì)', '公司規(guī)模', '所屬類別', '公司介紹']


def read_page(url, page_num, keyword): # 模仿瀏覽器post需求信息，并讀取返回后的頁面信息
  page_headers = {
    'Host': 'www.lagou.com',
    'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) '
           'Chrome/45.0.2454.85 Safari/537.36 115Browser/6.0.3',
    'Connection': 'keep-alive'
    }
  if page_num == 1:
    boo = 'true'
  else:
    boo = 'false'
  page_data = parse.urlencode([  # 通過頁面分析，發(fā)現(xiàn)瀏覽器提交的FormData包括以下參數(shù)
    ('first', boo),
    ('pn', page_num),
    ('kd', keyword)
    ])
  req = request.Request(url, headers=page_headers)
  page = request.urlopen(req, data=page_data.encode('utf-8')).read()
  page = page.decode('utf-8')
  return page


def read_tag(page, tag):
  page_json = json.loads(page)
  page_json = page_json['content']['result'] # 通過分析獲取的json信息可知，招聘信息包含在返回的result當(dāng)中，其中包含了許多其他參數(shù)
  page_result = [num for num in range(15)] # 構(gòu)造一個容量為15的list占位，用以構(gòu)造接下來的二維數(shù)組
  for i in range(15):
    page_result[i] = [] # 構(gòu)造二維數(shù)組
    for page_tag in tag:
      page_result[i].append(page_json[i].get(page_tag)) # 遍歷參數(shù)，將它們放置在同一個list當(dāng)中
    page_result[i][8] = ','.join(page_result[i][8])
  return page_result  # 返回當(dāng)前頁的招聘信息


def read_max_page(page): # 獲取當(dāng)前招聘關(guān)鍵詞的最大頁數(shù)，大于30的將會被覆蓋，所以最多只能抓取30頁的招聘信息
  page_json = json.loads(page)
  max_page_num = page_json['content']['totalPageCount']
  if max_page_num > 30:
    max_page_num = 30
  return max_page_num


def save_excel(fin_result, tag_name, file_name): # 將抓取到的招聘信息存儲到excel當(dāng)中
  book = xlsxwriter.Workbook(r'C:\Users\Administrator\Desktop\%s.xls' % file_name) # 默認(rèn)存儲在桌面上
  tmp = book.add_worksheet()
  row_num = len(fin_result)
  for i in range(1, row_num):
    if i == 1:
      tag_pos = 'A%s' % i
      tmp.write_row(tag_pos, tag_name)
    else:
      con_pos = 'A%s' % i
      content = fin_result[i-1] # -1是因?yàn)楸槐砀竦谋眍^所占
      tmp.write_row(con_pos, content)
  book.close()


if __name__ == '__main__':
  print('**********************************即將進(jìn)行抓取**********************************')
  keyword = input('請輸入您要搜索的語言類型：')
  fin_result = [] # 將每頁的招聘信息匯總成一個最終的招聘信息
  max_page_num = read_max_page(read_page(url, 1, keyword))
  for page_num in range(1, max_page_num):
    print('******************************正在下載第%s頁內(nèi)容*********************************' % page_num)
    page = read_page(url, page_num, keyword)
    page_result = read_tag(page, tag)
    fin_result.extend(page_result)
  file_name = input('抓取完成，輸入文件名保存：')
  save_excel(fin_result, tag_name, file_name)
  endtime = datetime.datetime.now()
  time = (endtime - starttime).seconds
  print('總共用時：%s s' % time)