今天在爬职位信息的时候，先用的requests模块，后来发现，获取的数据全部是乱码。果断换成selenium+Chrome,成功获取数据，保存MongoDB
在前期校验获取成果时，建议用selenium+Chrome，后期测试通过，换成无界面的PhantomJS
代码如下：
import timefrom selenium import webdriverfrom selenium.webdriver.common.by import Byfrom selenium.webdriver.support.ui import WebDriverWaitfrom selenium.webdriver.support import expected_conditions as ECfrom pyquery import PyQuery as pq# from config import *import pymongobrowser = webdriver.Chrome('/home/worker/Desktop/driver/chromedriver')wait = WebDriverWait(browser, 10)browser.set_window_size(1400, 900)# 连接数据库client = pymongo.MongoClient('127.0.0.1', 27017)# 定义数据库名称db = client.Job# 定义表名coll = db.jobfor i in range(1,101):browser.get('https://search.51job.com/list/020000,000000,0000,00,9,99,Python,2,{}.html?lang=c&stype=1&postchannel=0000&workyear=99&cotype=99°reefrom=99&jobterm=99&companysize=99&lonlat=0,0&radius=-1&ord_field=0&confirmdate=9&fromType=4&dibiaoid=0&address=&line=&specialarea=00&from=&welfare='.format(i)) wait.until( EC.presence_of_element_located((By.CSS_SELECTOR, '#resultList > div')) ) html = browser.page_source # print(html) doc = pq(html) #siblings,在兄弟元素中找('.el')的元素 items = doc('#resultList .title').siblings('.el').items() #当获取数据时，有的字段有数据有的没有，把获取数据之前的页面保存，比对 # with open('./html.html','w') as f: # f.write(html) for item in items: # for row in rows: Job={} # Job['job_name']=item.find('a').eq(0).text(), # eq(0),想得到第一个标签内的内容 Job['job_name']=item.find('.t1').eq(0).text() Job['com_name']=item.find('.t2').text() Job['addr']=item.find('.t3').text() try: Job['job_money']=item.find('.t4').text() except: Job['job_money']=None Job['job_time']=item.find('.t5').text() # time.sleep(0.1) # print(Job['job_name']) # print(Job['com_name']) coll.insert(Job)
# print(Job)
