代碼：

program: csdn博文爬蟲
function: 實現(xiàn)對我的csdn主頁所有博文的日期、主題、訪問量、評論個數(shù)信息爬取
version: python 3.5.1
time: 2016/05/29
author: yr

import urllib.request,re,time,random,gzip

定義保存文件函數(shù)

def saveFile(data,i):

path = "E:\\projects\\Spider\\05_csdn\\papers\\paper_"+str(i+1)+".txt"
file = open(path,'wb')
page = '當(dāng)前頁：'+str(i+1)+'\n'
file.write(page.encode('gbk'))
#將博文信息寫入文件(以utf-8保存的文件聲明為gbk)
for d in data:
    d = str(d)+'\n'
    file.write(d.encode('gbk'))
file.close()

解壓縮數(shù)據(jù)

def ungzip(data):

try:
    #print("正在解壓縮...")
    data = gzip.decompress(data)
    #print("解壓完畢...")
except:
    print("未經(jīng)壓縮，無需解壓...")
return data

CSDN爬蟲類

class CSDNSpider:

def __init__(self,pageIdx=1,url="http://blog.csdn.net/fly_yr/article/list/1"):
    #默認當(dāng)前頁
    self.pageIdx = pageIdx
    self.url = url[0:url.rfind('/') + 1] + str(pageIdx)
    self.headers = {
        "Connection": "keep-alive",
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 "
                      "(KHTML, like Gecko) Chrome/51.0.2704.63 Safari/537.36",
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
        "Accept-Encoding": "gzip, deflate, sdch",
        "Accept-Language": "zh-CN,zh;q=0.8",
        "Host": "blog.csdn.net"
    }

#求總頁數(shù)
def getPages(self):
    req = urllib.request.Request(url=self.url, headers=self.headers)
    res = urllib.request.urlopen(req)

    # 從我的csdn博客主頁抓取的內(nèi)容是壓縮后的內(nèi)容，先解壓縮
    data = res.read()
    data = ungzip(data)
    data = data.decode('utf-8')

    pages = r'<div.*?pagelist">.*?<span>.*?共(.*?)頁</span>'
    #link = r'<div.*?pagelist">.*?<a.*?href="(.*?)".*?</a>'
    # 計算我的博文總頁數(shù)
    pattern = re.compile(pages, re.DOTALL)
    pagesNum = re.findall(pattern, data)
    return pagesNum

#設(shè)置要抓取的博文頁面
def setPage(self,idx):
    self.url = self.url[0:self.url.rfind('/')+1]+str(idx)

#讀取博文信息
def readData(self):
    ret=[]
    str = r'<div.*?article_item">.*?<span class="(.*?)"></span>.*?link_title"><a href="(.*?)">(.*?)</a>.*?' + \
          r'<span class="link_postdate">(.*?)</span>.*?</a>(.??)(.??)</span>.*?' + \
          r'</a>.*?(.??)(.??)</span>' 
    req = urllib.request.Request(url=self.url, headers=self.headers)
    res = urllib.request.urlopen(req)

    # 從我的csdn博客主頁抓取的內(nèi)容是壓縮后的內(nèi)容，先解壓縮
    data = res.read()
    data = ungzip(data)
    data = data.decode('utf-8')
    pattern = re.compile(str,re.DOTALL)
    items = re.findall(pattern,data)
    for item in items:  
        if item[0] == "ico ico_type_Original":  
            s = '原'  
        else:  
            s = '轉(zhuǎn)'  
        ret.append('\r\n日期：' + item[3] + '\r\t' + s  
                   + '\r\n標(biāo)題：' + item[2].strip()  
                   + '\r\n鏈接：http://blog.csdn.net' + item[1]  
                   + '\r\n閱讀：' + item[4] + '\r\t評論：' + item[5] + '\r\n')  
    return ret  

定義爬蟲對象

cs = CSDNSpider()

求取

pagesNum = int(cs.getPages())
print("博文總頁數(shù)： ",pagesNum)

for idx in range(pagesNum):

cs.setPage(idx)
print("當(dāng)前頁：",idx+1)
#讀取當(dāng)前頁的所有博文，結(jié)果為list類型
papers = cs.readData()
saveFile(papers,idx)

報錯：
Traceback (most recent call last):
File "I:/python_work/pachong/request www.py", line 99, in <module>

pagesNum = int(cs.getPages())

File "I:/python_work/pachong/request www.py", line 63, in getPages

pagesNum = re.findall(pattern, data)[0]

IndexError: list index out of range

小白求解