from lxml
import etree
from multiprocessing.dummy
import Pool
as ThreadPool
import requests
import json
import sys
reload(sys)
sys.setdefaultencoding(
'utf-8')
'''重新运行之前请删除content.txt，因为文件操作使用追加方式，会导致内容太多。'''
def towrite(contentdict):
f.writelines(
u'回帖时间:' + str(contentdict[
'topic_reply_time']) +
'\n')
f.writelines(
u'回帖内容:' + contentdict[
'topic_reply_content'] +
'\n')
f.writelines(
u'回帖人:' + contentdict[
'user_name'] +
'\n\n')
def spider(url):
html = requests.get(url)
selector = etree.HTML(html.text)
content_field = selector.xpath(
'//*[@class="l_post j_l_post l_post_bright "]')
item = {}
for each
in content_field:
tmp1=each.xpath(
'@data-field')
print type(tmp1[
0])
tmp=each.xpath(
'@data-field')[
0].replace(
'"',
'')
reply_info = json.loads(each.xpath(
'@data-field')[
0].replace(
'"',
''))
author = reply_info[
'author'][
'user_name']
content = each.xpath(
'div[@class="d_post_content_main"]/div/cc/div[@class="d_post_content j_d_post_content clearfix"]/text()')[
0]
reply_time = reply_info[
'content'][
'date']
print content
print reply_time
print author
item[
'user_name'] = author
item[
'topic_reply_content'] = content
item[
'topic_reply_time'] = reply_time
towrite(item)
if __name__ ==
'__main__':
pool = ThreadPool(
4)
f = open(
'content.txt',
'w')
page = []
for i
in range(
1,
21):
newpage =
'http://tieba.baidu.com/p/3522395718?pn=' + str(i)
page.append(newpage)
results = pool.map(spider, page)
pool.close()
pool.join()
f.close()
