抓取团购信息
myheader<-c(
"User-Agent"="Mozilla/5.0 (Linux; U; Android 1.0; en-us; dream) AppleWebKit/525.10 (KHTML, like Gecko) Version/3.0.4 Mobile Safari/523.12.2 ",
"Accept-Language"="en-us",
"Connection"="keep-alive",
"Accept-Charset"="GB2312,utf-8;q=0.7,*;q=0.7")#伪装header，防止不能爬取
temp<-getURL("http://t.dianping.com/list/nanjing-category_1",httpheader=myheader,encoding="UTF-8")#获取链接
write.table(temp,"temp.txt")#查看
temp1<-getURL("http://t.dianping.com/list/nanjing-category_1?pageIndex=1",httpheader=myheader,encoding="UTF-8")
write.table(temp,"temp1.txt")#查看下一页，判断能不能爬取
k=htmlParse(temp)#解析
k
youhui=sapply(getNodeSet(k,'//a [@class=\"tg-floor-title\"]'),xmlValue)#从获取的网页信息中截取出所需要的，查看源文件如下：
<a class=\"tg-floor-title\" target=\"_blank\" href=\"/deal/21919506\"
data-hippo-track=\"|1|module#5_con_list,action#click,index#2,dealgrp_id#21919506,query_id#c682fa87-6ceb-4423-8d9e-b65720eac1de\"
>
<h3>大渝火锅</h3>
<h4> [16店通用]
仅售90元！价值100元的代金券1张，除酒水饮料、调料外全场通用，可叠加使用，可免费使用包间，提供免费WiFi。</h4>
</a>
可以看到大概都在<a class=></a>中保存，所以用'//a [@class=\"tg-floor-title\"]，@代表class是个属性，并且用中括号，规定就是这样写。
youhui[30]#取出一个看看
write.table(youhui,"meishiyouhui.txt")#写入一个文件查看一下
#循环取出7页来
urllist=0
page=1:7
urllist[page]=paste("http://t.dianping.com/list/nanjing-category_1?pageIndex=",page,sep='')#设定这7页的网页链接，观察网页链接可以得到这个规律，循环的过程跟上面的过程一致
for (url in urllist){
temp=getURL(url,httpheader=myheader,encoding="UTF-8")
k=htmlParse(temp)
youhui=sapply(getNodeSet(k,'//a [@class=\"tg-floor-title\"]'),xmlValue)
cat(url,"\n")
write.table(youhui,"meishi.txt",append=T)#append=T,这样读取7页的信息就不会重复覆盖
}
#随机设定伪装头的方法，定义一个列表，从里面随机选取一个作为user-agent
#大众点评美食团购页面，测试另外一种伪装头的方法 library(RCurl) library(XML) UserAgent=c( "Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1", "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.16 Safari/537.36", "Mozilla/5.0 (Windows NT 6.1; Intel Mac OS X 10.6; rv:7.0.1) Gecko/20100101 Firefox/7.0.1", "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36 OPR/18.0.1284.68", "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)", "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Trident/6.0)", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:7.0.1) Gecko/20100101 Firefox/7.0.1", "Opera/9.80 (Macintosh; Intel Mac OS X 10.9.1) Presto/2.12.388 Version/12.16", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36 OPR/18.0.1284.68", "Mozilla/5.0 (iPad; CPU OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) CriOS/30.0.1599.12 Mobile/11A465 Safari/8536.25", "Mozilla/5.0 (iPad; CPU OS 8_0 like Mac OS X) AppleWebKit/600.1.3 (KHTML, like Gecko) Version/8.0 Mobile/12A4345d Safari/600.1.4", "Mozilla/5.0 (iPad; CPU OS 7_0_2 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A501 Safari/9537.53", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50", "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0" ) n=length(UserAgent) myheader<-c( "User-Agent"=UserAgent[sample(n-1,1)],#随机选择 "Accept-Language"="en-us", "Connection"="keep-alive", "Accept-Charset"="GB2312,utf-8;q=0.7,*;q=0.7") urllist=0 page=1:3 urllist[page]=paste("http://t.dianping.com/list/nanjing-category_1?pageIndex=",page,sep='') for(url in urllist){ temp<-getURL(url,encoding="UTF-8",httpheader=myheader)# k=htmlParse(temp) review=sapply(getNodeSet(k,'//a [@class=\"tg-floor-title\"]'),xmlValue) review[page] cat(url,"\n") write.table(review,"dazhongdianping.txt",quote = FALSE,row.names = TRUE, col.names = FALSE,append=T) #Sys.sleep(5) }
