最近由于参加学校举办短时速学python的比赛，学习了一遍python这门语言，原来一直认为Java语言是最牛逼的，现在发现python也有它的可取之处，它开发快，语言简洁，对于数组的处理，让我发现利用它开发一些简单的程序真的比java快^^ 下面，记录一下我利用python实现爬虫，获取百度文库词条其中包含”python”信息的样例（技术Python、Mysql）
1、爬虫架构，以及原理 爬虫重要的架构有三个，分别是URL管理器、网页下载器、网页解析器，还有一个调度器、数据输出器 URL管理器：管理所有的URL，负责取出URL给网页下载器，并将该URL设定为以爬取 网页下载器：将互联网对应的网页下载到本地 网页解析器：解析网页中重要的数据信息（在本样例中为词条信息），并且从该网页中获取其他符合要求的URL，存入Mysql，以便URL管理器取 调度器：类似与java中的main方法，相当于开启爬虫的入口，它负责初始化第一个入口URL(地址)，利用while循环依次调用URL管理器、网页下载器、网页解析器完成相关功能。 数据输出器：将得到数据输出 如下图：
2、代码框架 1、利用Mysql数据库， 数据库表baike_spider，账户：root,密码：0203
CREATE TABLE `baike_spider` ( `webSite` varchar(255) DEFAULT NULL, `isCraw` int(1) DEFAULT '0', `title` varchar(255) DEFAULT NULL, `cont` text, KEY `webSide` (`webSite`) USING HASH ) ENGINE=InnoDB DEFAULT CHARSET=utf8;2、源程序框架
3、爬取结果展示： 尴尬，显示的python就两三个。。 不过没关系，肯定在没显示出来的地方^_^.
3、调度器 # coding=utf-8 # import baike_sider import url_manager, html_donloader, html_parse, html_outputer import sys default_encoding = 'utf-8' if sys.getdefaultencoding() != default_encoding: reload(sys) sys.setdefaultencoding(default_encoding) class SpiderMain(object): def __init__(self): # 初始化四个管理器 self.urls = url_manager.UrlManager() self.donloader = html_donloader.HtmlDonload() self.parse = html_parse.HtmlParse() self.outputer = html_outputer.HtmlOutputer() def craw(self, root_url): # 初始化urls管理器，赋初值 self.urls.init_url(root_url) # 计算一个count来存储爬虫收集的网站个数 count = 0 # 当url管理器中还存在未被搜刮的网站，循环继续 while self.urls.has_new_url(): try: # 从URL管理器中获取一个新的url地址 new_url = self.urls.get_new_url() # 利用网站下载器将其下载下来 cont = self.donloader.donload(new_url) # 解析器解析cont网站，得到新的urls，和新的数据 urls, new_data = self.parse.parse(new_url, cont) # 将新的地址存入URL管理器 self.urls.add_new_urls(urls) # 输出管理器收集这些新信息 if new_data is not None: self.outputer.collect_data(new_url, new_data) # 如果爬虫搜刮的网站个数到达1000，停止运行 print "爬完%s,得到%d个新网站，得到信息%s,%s" % (new_url,len(urls),new_data.values()[0],new_data.values()[1]) print "正在爬第%d个网站" % count if count == 1000: break # 统计网站个数加一 count += 1 except Exception,value: print "craw error :",value # 返回输出管理器 # return self.outputer print "craw完毕" pass __name__ = "_main_" if __name__ == "_main_": root_url = "http://baike.baidu.com/item/Python" obj_spider = SpiderMain() obj_spider.craw(root_url) 4、URL管理器 # coding=utf-8 import MySQLdb as mdb # try: # cursor = self.db.cursor() # # self.db.commit() # except Exception,value: # self.db.rollback() # print "URLManager.__init__url: ",value # finally: # cursor.close() class UrlManager(object): def __init__(self): self.db = mdb.connect("localhost","root","0203","bigData",charset="utf8") cursor = self.db.cursor() delete_sql = '''drop table if exists baike_spider''' create_sql = '''create table if not exists baike_spider( webSite varchar(255), isCraw int(1) default '0', title varchar(255), cont text, KEY `webSide` (`webSite`) USING HASH )''' try: cursor.execute(delete_sql) cursor.execute(create_sql) cursor.execute("SET NAMES UTF8") self.db.commit() except Exception,value: print "URLManager.__init__Error: ",value self.db.rollback() finally: cursor.close() pass def init_url(self, root_url): try: cursor = self.db.cursor() cursor.execute("SET NAMES UTF8") insert_sql = '''insert into baike_spider(webSite) values('%s')''' % root_url cursor.execute(insert_sql) self.db.commit() except Exception,value: self.db.rollback() print "URLManager.__init__url: ",value finally: cursor.close() pass def has_new_url(self): new = 0 try: cursor = self.db.cursor() cursor.execute("SET NAMES UTF8") select_sql = '''select isCraw from baike_spider where isCraw=0 limit 1''' new = cursor.execute(select_sql) except Exception,value: print "URLManager.has_new_url: ",value finally: cursor.close() # print 'new=',new return new pass def get_new_url(self): url = "" try: cursor = self.db.cursor() cursor.execute("SET NAMES UTF8") select_sql = '''select * from baike_spider where isCraw=0 limit 1''' cursor.execute(select_sql) url = cursor.fetchone()[0] update_sql = '''update baike_spider set isCraw=1 where webSite='%s' ''' cursor.execute(update_sql % url) self.db.commit() except Exception,value: self.db.rollback() print "URLManager.has_new_url: ",value finally: cursor.close() return url def add_new_urls(self,urls): is_exist = '''select isCraw from baike_spider where webSite='%s' ''' insert_sql = '''insert into baike_spider(webSite) values('%s')''' try: cursor = self.db.cursor() cursor.execute("SET NAMES UTF8") for url in urls: flag = cursor.execute(is_exist % url) if flag:continue else: cursor.execute(insert_sql % url) self.db.commit() except Exception,value: print "URLManager.add_new_urls: ",value self.db.rollback() finally: cursor.close() pass # urlManage = UrlManager() # urlManage.has_new_url() # urls = ["http://www.baidu.com","http://www.baidu.com4","http://www.baidu.com2","http://www.baidu.com1","http://www.baidu.com3"] # print urlManage.add_new_urls(urls) 5、网页下载器 # coding=utf-8 import urllib2 class HtmlDonload(): def __init__(self): pass def donload(self, url): cont = "" try: response = urllib2.urlopen(url) if response.getcode()==200: #读取网页内容 cont = response.read() #输出内容 # print cont except Exception,value: print "HtmlDonload(),Error",value return cont pass # HtmlDonload().donload("http://www.baidu.com") 6、网页解析器 # coding=utf-8 import re import urlparse from bs4 import BeautifulSoup class HtmlParse(): def __init__(self): pass def _get_new_urls(self, page_url, soup): new_urls = set() links = soup.find_all(name='a', href=re.compile(r"/item/")) for link in links: new_url = link['href'] new_full_url = urlparse.urljoin(page_url, new_url) # print "new_full_url = ",new_full_url new_urls.add(new_full_url) return new_urls def _get_new_data(self, url, soup): res_data = {} title_node = soup.find("dd", class_="lemmaWgt-lemmaTitle-title").find("h1") res_data["title",] = title_node.get_text() summary_node = soup.find("div", class_="lemma-summary") res_data["summary"] = summary_node.get_text() # print "res_data = ", res_data return res_data def parse(self, url, cont): if cont is None or url is None: return # 将cont传入生成一个beautifulSoup对象 soup = BeautifulSoup(cont, 'html.parser', from_encoding='utf-8') new_urls = self._get_new_urls(url, soup) new_data = self._get_new_data(url, soup) return new_urls, new_data # parse = HtmlParse() # parse.parse("baidu.html",open("hello.html")) 7、数据输出器 # coding=utf-8 import MySQLdb as mdb class HtmlOutputer(): def __init__(self): self.db = mdb.connect("localhost","root","0203","bigData",charset="utf8") pass def collect_data(self, url, new_data): try: cursor = self.db.cursor() cursor.execute("SET NAMES UTF8") insert_sql = '''update baike_spider set title='%s',cont='%s' where webSite='%s' ''' data = new_data.values() cursor.execute(insert_sql % (data[0],data[1],url)) self.db.commit() except Exception,value: self.db.rollback() print "HtmlOutputer.collect_data: ",value finally: cursor.close() pass def print_data(self): print 123 try: cursor = self.db.cursor() cursor.execute("SET NAMES UTF8") insert_sql = '''select * from baike_spider where isCraw=1 ''' cursor.execute(insert_sql) results = cursor.fetchall() for result in results: print result[2],result[3] self.db.commit() except Exception,value: self.db.rollback() print "HtmlOutputer.collect_data: ",value finally: cursor.close() pass