#-*- conding:utf-8 -*-
import requests,re,pymysql
from pyquery import PyQuery as pq


url_index = ['http://finance.eastmoney.com/news/cgnjj_{}.html'.format(pages) for pages in range(1,2)]

def index_content(url_index,title=0,time=0):
        #print('起始頁：' + url_index)
        index_req = requests.get(url_index,timeout = 10)
        index_req.encoding = 'utf-8'
        index_content = index_req.text
        content_url = re.findall('<div class="text text-no-img">.*?<p class="title">.*?<a href="(.*?)" target="_blank">',index_content,re.S)
        title = re.search('<p class="title">.*?<a href=".*?" target="_blank">(.*?)</a>',index_content,re.S).group(1)
        time = re.search('<p class="time">(.*?)</p>',index_content,re.S).group(1).replace("年", "-").replace("月", "-").replace("日", "")
        #time = time_content
        #re.sub(".", "", title)
        #print('標(biāo)題：' + title)
        #print('時(shí)間：' + time)
        return content_url
        
def content(content_url):
        content_req = requests.get(content_url,timeout = 10)
        content_req.encoding = 'utf-8'
        content = content_req.text
        title = re.search('<h1>(.*?)</h1>',content,re.S).group(1)
        time = re.search('<div class="time">(.*?)</div>',content,re.S).group(1).replace("年", "-").replace("月", "-").replace("日", "")
        sources = re.search('<div class="source">.*?<span>來源：</span>(.*?)</div>',content,re.S).group(1)
        content_content = re.search('<!--文章主體-->(.*?)<!--責(zé)任編輯-->',content,re.S).group(1)
        pic = re.findall('''.*?src="https://z1.dfcfw.com(.*?)"''',content_content,re.S)#.sub('.*?same.*?','',content_content)
        content_body = {'title:':title,
                        'time:':time,
                        'source:':sources,
                        'content:':content_content,
                        'pic:':pic
            
        }
#         print('網(wǎng)址：' + content_url)
#         print ('標(biāo)題：' + title)
#         print ('時(shí)間：' + time)
#         print ('來源:' + source)
#         print ('內(nèi)容：' + content_content )
#         print (pic)
        print(type(title))
        return content_body
        
def save_sql(content_body):
    try:
        db = pymysql.connect("localhost","root","root","test" )
        cursor = db.cursor()
        #sql = """INSERT INTO test(title,source_time, source, content_body, pageurl,pic)VALUES (title,time,source,content_content,content_url,pic)"""
        # 執(zhí)行sql語句
        sql = """INSERT INTO test(title,source_time, sources, content_body, pageurl,pic)VALUES (%s,%s,%s,%s,%s,%s)""" 
# 執(zhí)行 sql 語句 
        cursor.execute(sql,(title,time,sources,content_content,content_url,pic))
        #cursor.execute(sql)
        # 提交到數(shù)據(jù)庫執(zhí)行
        db.commit()
    except:
        # 如果發(fā)生錯誤則回滾
        db.rollback()
        # 關(guān)閉數(shù)據(jù)庫連接
        db.close()

def save_csv(con):
    with open('C:\\Users\\Administrator\\Desktop\\finance.csv','wb+') as f: #wb為以二進(jìn)制方式打開
        con = str.encode('utf-8')
        f.write(con)
        f.close()

for url_list in url_index:
    content_url = index_content(url_list,title=0,time=0)
    for con_url in content_url:
        cons = content(con_url)
        print(cons)
        save_sql(cons)
#         for con in cons:
#             #print(con)
#             save_csv(con)
        #save_sql(con)

運(yùn)行沒報(bào)錯，也能正常打印輸出，但是就是保存不到數(shù)據(jù)庫...而且保存到文件的時(shí)候，老是提示類型不對，字典、元組、字符串都不行，幫忙看下..