# -*- coding: utf-8 -*-
import re
import urllib2
from collections import deque
queue = deque()
visited = set()
url = '入口页面, 可以换成别的
req_header = {'User-Agent':'Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; rv:11.0) like Gecko'}
req_timeout = 20
queue.append(url)
cnt = 0
while queue:
url = queue.popleft() # 队首元素出队
visited |= {url} # 标记为已访问
print(u'已经抓取: ' + str(cnt) + u' 正在抓取 <--- ' + url)
cnt += 1
req = urllib2.Request(url,None,req_header)
urlop = urllib2.urlopen(req,None,req_timeout)
try:
data = urlop.read().decode('utf-8')
except:
continue
# 正则表达式提取页面中所有队列, 并判断是否已经访问过, 然后加入待爬队列
linkre = re.compile('href=\"(.+?)\"')
for x in linkre.findall(data):
print(x)
if 'http' in x and x not in visited:
queue.append(x)
print(u'加入队列 ---> ' + x)