- 主机环境：Ubuntu 13.04
- Python版本：2.7.4
- Django版本：1.5.4
- Scrapy版本：0.18.2
- ElasticSearch版本：0.90.5
闲来无聊，查看了相关搜索引擎的基本知识，经过搜集资料，了解了搜索引擎所需要的基本子系统，爬取子系统，索引服务子系统，Web请求和应答子系统。然后经过学习基本的开源框架文档，集成的项目已经PUSH到GitHub。
首先查看基于开源的Scrapy爬虫框架编写的一个爬虫，爬取校园网的内容（主要是免流量）
01 |
#!/usr/bin/env python |
02 |
#-*- coding:utf-8 -*- |
03 |
#from urlparse import urljoin |
04 |
from scrapy.utils.url import urljoin_rfc |
05 |
from scrapy.spider import BaseSpider |
06 |
from scrapy.selector import HtmlXPathSelector |
07 |
from scrapy.http import Request |
08 |
|
09 |
from scrapy.exceptions import DropItem |
10 |
|
11 |
from mymodules.items import Website |
12 |
|
13 |
import urllib |
14 |
import re |
15 |
|
16 |
class Xidian_Spider(BaseSpider): |
17 |
name = "xidian_spider" |
18 |
start_urls = [ |
19 |
"http://www.xidian.edu.cn", |
21 |
|
22 |
] |
23 |
|
24 |
def __init__(self): |
25 |
"""init the allowed_domain""" |
26 |
self.allowed_domains = ['xidian.edu.cn'] |
27 |
|
28 |
def parse(self, response): |
29 |
"""In this parse,we use double yeild to return the item or Request""" |
30 |
hxs = HtmlXPathSelector(response) |
31 |
|
32 |
refer_websites = hxs.select('//@href').extract() |
33 |
|
34 |
#if not self.gethostname(response.url) in self.allowed_domains: |
35 |
# self.allowed_domains.append(self.gethostname(response.url)) |
36 |
|
37 |
item = Website() |
38 |
item['url'] = response.url |
39 |
item['title'] = hxs.select('/html/head/title/text()').extract()[0] |
40 |
|
41 |
"""FIXME:This XPath select all the elements,include the javascript code.BAD!!""" |
42 |
str = '' |
43 |
list = hxs.select('/html/body//*/text()').extract() |
44 |
for s in list: |
45 |
str += s.strip() |
46 |
str += ' ' |
47 |
|
48 |
item['content'] = str |
49 |
|
50 |
yield item |
51 |
|
52 |
for weburl in refer_websites: |
53 |
|
54 |
utf8_url = weburl.encode('utf-8') |
55 |
|
56 |
"""The following regex to match the prefix and postfix of urls""" |
57 |
postfix = re.compile(r'.+\.((jpg)|(ico)|(rar)|(zip)|(doc)|(ppt)|(xls)|(css)|(exe)|(pdf))x?$') |
58 |
prefix = re.compile(r'^((javascript:)|(openapi)).+') |
59 |
|
60 |
if postfix.match(utf8_url): |
61 |
continue |
62 |
if prefix.match(utf8_url): |
63 |
continue |
64 |
if not utf8_url.startswith('http://'): |
65 |
#weburl = urljoin_rfc(response.url, weburl, response.encoding) |
66 |
weburl = 'http://'+self.gethostname(response.url)+'/'+weburl |
67 |
|
68 |
weburl = re.sub(r'/\.\./\.\./',r'/',weburl) |
69 |
weburl = re.sub(r'/\.\./',r'/',weburl) |
70 |
|
71 |
yield Request(weburl, callback=self.parse) |
72 |
|
73 |
def gethostname(self, res_url): |
74 |
"""get the host name of a url""" |
75 |
proto, rest = urllib.splittype(res_url) |
76 |
host, rest = urllib.splithost(rest) |
77 |
return host |
爬取得到的ITEM会交给PIPELINE处理。
这里的PipeLine做了去重处理，不能简单的放在内容，所以使用的是Bloom Filter的算法，这里直接安装了Python的开源库中的pybloomfilter（有时间研究一下）
01 |
class DuplicatesPipeline(object): |
02 |
|
03 |
def __init__(self): |
04 |
self.bf = BloomFilter(10000000, 0.01, 'filter.bloom') |
05 |
self.f_write = open('visitedsites','w') |
06 |
self.si = SearchIndex() |
07 |
self.si.SearchInit() |
08 |
|
09 |
def process_item(self, item, spider): |
10 |
print '************%d pages visited!*****************' %len(self.bf) |
11 |
if self.bf.add(item['url']):#True if item in the BF |
12 |
raise DropItem("Duplicate item found: %s" % item) |
13 |
else: |
14 |
#print '%d pages visited!'% len(self.url_seen) |
15 |
self.save_to_file(item['url'],item['title']) |
16 |
self.si.AddIndex(item) |
17 |
return item |
18 |
|
19 |
def save_to_file(self,url,utitle): |
20 |
self.f_write.write(url) |
21 |
self.f_write.write('\t') |
22 |
self.f_write.write(utitle.encode('utf-8')) |
23 |
self.f_write.write('\n') |
24 |
|
25 |
def __del__(self): |
26 |
"""docstring for __del__""" |
27 |
self.f_write.close() |
28 |
self.si.IndexDone() |
该类中的SearchIndex是ElasticSearch建立索引的类。定义如下：
01 |
#!/usr/bin/env python |
02 |
#-*- coding:utf-8-*- |
03 |
import os |
04 |
import sys |
05 |
from pyes import * |
06 |
from mymodules.items import Website |
07 |
INDEX_NAME='xidian_spider' |
08 |
|
09 |
class SearchIndex(object): |
10 |
|
11 |
def SearchInit(self): |
12 |
self.conn = ES('127.0.0.1:9200', timeout=3.5)#Connect to ES |
13 |
try: |
14 |
self.conn.delete_index(INDEX_NAME) |
15 |
#pass |
16 |
except: |
17 |
pass |
18 |
self.conn.create_index(INDEX_NAME)#Create a new INDEX |
19 |
|
20 |
#Define the structure of the data format |
21 |
mapping = {u'content': {'boost': 1.0, |
22 |
'index': 'analyzed', |
23 |
'store': 'yes', |
24 |
'type': u'string', |
25 |
"indexAnalyzer":"ik", |
26 |
"searchAnalyzer":"ik", |
27 |
"term_vector" : "with_positions_offsets"}, |
28 |
u'title': {'boost': 1.0, |
29 |
'index': 'analyzed', |
30 |
'store': 'yes', |
31 |
'type': u'string', |
32 |
"indexAnalyzer":"ik", |
33 |
"searchAnalyzer":"ik", |
34 |
"term_vector" : "with_positions_offsets"}, |
35 |
u'url': {'boost': 1.0, |
36 |
'index': 'analyzed', |
37 |
'store': 'yes', |
38 |
'type': u'string', |
39 |
#"indexAnalyzer":"ik", |
40 |
#"searchAnalyzer":"ik", |
41 |
"term_vector" : "with_positions_offsets"}, |
42 |
} |
43 |
|
44 |
self.conn.put_mapping("searchEngine-type", {'properties':mapping}, [INDEX_NAME])#Define the type |
45 |
|
46 |
def AddIndex(self,item): |
47 |
|
48 |
print 'Adding Index item URL %s'% item['title'].encode('utf-8') |
49 |
self.conn.index({'title':item['title'].encode('utf-8'), \ |
50 |
'url':item['url'].encode('utf-8'),\ |
51 |
'content':item['content'].encode('utf-8')\ |
52 |
},INDEX_NAME,'searchEngine-type') |
53 |
|
54 |
def IndexDone(self): |
55 |
self.conn.default_indices=[INDEX_NAME]#Set the default indices |
56 |
self.conn.refresh()#Refresh the ES |
其中中文分词使用的是IK分词，Python库中直接安装即可。
Django中接受搜索请求的处理函数如下：
01 |
def search(request): |
02 |
"""docstring for search""" |
03 |
if 'q' in request.GET: |
04 |
q = request.GET['q'] |
05 |
print q |
06 |
if 'page' in request.GET: |
07 |
page = unicode(request.GET['page']) |
08 |
else: |
09 |
page = unicode(1) |
10 |
start = clock() |
11 |
results = dosearch(q,page)#connect to ES to return the results |
12 |
end = clock() |
13 |
return render(request,'res_search.html', {'results' : results, |
14 |
'query':q, |
15 |
'count':len(results), |
16 |
'time':end-start, |
17 |
'page':page, |
18 |
'nextpage':int(page)+1}) |
19 |
else: |
20 |
message = 'You submitted an empty form.' |
21 |
return HttpResponse(message) |
其中调用dosearch函数进行连接ES查询，函数内容如下：
01 |
def dosearch(string,upage): |
02 |
conn = ES('127.0.0.1:9200', timeout=3.5)#连接ES |
03 |
fq_title = FieldQuery(analyzer='ik') |
04 |
fq_title.add('title',string) |
05 |
|
06 |
fq_content = FieldQuery(analyzer='ik') |
07 |
fq_content.add('content',string) |
08 |
|
09 |
bq = BoolQuery(should=[fq_title,fq_content]) |
10 |
|
11 |
h=HighLighter(['['], [']'], fragment_size=100) |
12 |
|
13 |
page = int(upage.encode('utf-8')) |
14 |
if page < 1: |
15 |
page = 1 |
16 |
|
17 |
s=Search(bq,highlight=h,start=(page-1)*PAGE_SIZE,size=PAGE_SIZE) |
18 |
s.add_highlight("content") |
19 |
s.add_highlight('title') |
20 |
results=conn.search(s,indices='xidian_spider',doc_types='searchEngine-type') |
21 |
|
22 |
list=[] |
23 |
for r in results: |
24 |
if(r._meta.highlight.has_key("title")): |
25 |
r['title']=r._meta.highlight[u"title"][0] |
26 |
if(r._meta.highlight.has_key('content')): |
27 |
r['content']=r._meta.highlight[u'content'][0] |
28 |
|
29 |
res = Results() |
30 |
res.content = r['content'] |
31 |
res.title = r['title'] |
32 |
res.url = r['url'] |
33 |
list.append(res) |
34 |
return list |