求助：多進(jìn)程調(diào)用百度地圖 API 獲取數(shù)據(jù)，比單進(jìn)程慢，為什么?。ǜ酱a）？

多進(jìn)程調(diào)用百度地圖 api，先獲取經(jīng)緯度，然后利用經(jīng)緯度獲取過路費，保存到 excel （獲取一條存一條），因為想加快速度，所以寫了多進(jìn)程，但是經(jīng)過測試，多進(jìn)程比單進(jìn)程還要慢，想請教下為什么??？附代碼（我的 ak 隱藏了）：

單進(jìn)程

import requests
from openpyxl import load_workbook
import time

# 獲取經(jīng)緯度
def geocode(address):
    base = url = "http://api.map.baidu.com/geocoder/v2/?address=" + address + "&output=json&ak="
    response = requests.get(base)
    answer = response.json()
    return answer['result']['location']

# 獲取過路費
def get(origin_lat,origin_lng,destination_lat,destination_lng):
    base = url = "http://api.map.baidu.com/direction/v2/driving?origin=" + str(origin_lng) + "," + str(origin_lat) +  "&destination=" \
                 + str(destination_lng)+","+str(destination_lat) + "&output=json&ak="
    response = requests.get(base)
    answer = response.json()
    info = [answer['result']['routes'][0]['duration']/60,answer['result']['routes'][0]['distance']/1000,answer['result']['routes'][0]['toll']]
    return info

if __name__=='__main__':
    start = time.clock()
    data = load_workbook(r"ODdata.xlsx")
    table = data.get_sheet_by_name('locationcode')
    nrows = table.max_row
    ncols = table.max_column

    origin_table = data.get_sheet_by_name('OD')
    origin_nrows = origin_table.max_row
    origin_ncols = origin_table.max_column
    go_outset = []
    go_destination = []
    for r in range(2,nrows+1):
        go_outset.append(table.cell(row=r,column=2).value) #生成去程出發(fā)地列表
    for r in range(2,nrows+1):
        go_destination.append(table.cell(row=r,column=5).value) #生成去程目的地列表

    go_outset_count=1
    go_destination_count=1
    go_outset_locationcode = []
    for i in go_outset:
        try:
            go_outset_locationcode.append(geocode(i)) #生成去程出發(fā)地經(jīng)緯度
            print("出發(fā)地經(jīng)緯度查詢計數(shù)%d"%go_outset_count)
            go_outset_count+=1
        except:
            go_outset_locationcode.append({'lat':'wrong','lng':'wrong'})

    go_destination_locationcode = []
    for i in go_destination:
        try:
            go_destination_locationcode.append(geocode(i))#生成去程目的地經(jīng)緯度
            print("目的地經(jīng)緯度查詢計數(shù)%d" % go_destination_count)
            go_destination_count+=1
        except:
            go_destination_locationcode.append({'lat':'wrong','lng':'wrong'})

    go_outset_locationcodelist = []
    go_destination_locationcodelist = []

    for i in range(len(go_outset_locationcode)):
        go_outset_locationcodelist.append(go_outset_locationcode[i].values())

    for i in range(len(go_destination_locationcode)):
        go_destination_locationcodelist.append(go_destination_locationcode[i].values())

    #將經(jīng)緯度和省份寫入 excel
    for i in range(2,nrows+1):
        for j in range(3,5):
            _ = table.cell(column=j, row=i, value=list(go_outset_locationcodelist[i-2])[j-3])
    for i in range(2,nrows+1):
        for j in range(6,8):
            _ = table.cell(column=j, row=i, value=list(go_destination_locationcodelist[i-2])[j-6])
    data.save(r"ODdata.xlsx")

    #獲取過路費
    info = []
    go_count=1
    for i in range(0,len(go_outset)):
        if list(go_outset_locationcodelist[i])[0]=='wrong':
            continue
        else:
            try:
                info.append(get(list(go_outset_locationcodelist[i])[0],list(go_outset_locationcodelist[i])[1],list(go_destination_locationcodelist[i])[0],list(go_destination_locationcodelist[i])[1]))
                print("過路費查詢計數(shù)%d" % go_count)
                go_count+=1
            except:
                info.append(['wrong','wrong','wrong'])
                print("錯誤行數(shù)是%d"%i)
            finally:
                for j in range(8,11):
                    _ = origin_table.cell(column=j, row=i+3, value=info[i][j - 8])
                data.save(r"ODdata.xlsx")

    elapsed = (time.clock() - start)
    print("Time used:", elapsed)

多進(jìn)程

import requests
from openpyxl import load_workbook
import multiprocessing
from multiprocessing import Lock,Pool
import time


# 獲取經(jīng)緯度
def geocode(address):
    base = url = "http://api.map.baidu.com/geocoder/v2/?address=" + address + "&output=json&ak="
    response = requests.get(base)
    answer = response.json()
    return answer['result']['location']

# 保存到 excel
def save(info):
    # data_new = load_workbook(r"ODdata.xlsx")
    # origin_table_new = data_new['OD']
    for j in range(8, 11):
        _ = origin_table.cell(column=j, row=i + 3, value=info[j-8])
    data.save(r"ODdata.xlsx")
    print("第%d 行保存成功" % (i + 1))

# 獲取過路費
def getall(i,origin_lat,origin_lng,destination_lat,destination_lng):
    try:
        base = url = "http://api.map.baidu.com/direction/v2/driving?origin=" + str(origin_lng) + "," + str(
            origin_lat) + "&destination=" \
                     + str(destination_lng) + "," + str(
            destination_lat) + "&output=json&ak="
        response = requests.get(base)
        answer = response.json()
        info = [answer['result']['routes'][0]['duration'] / 60, answer['result']['routes'][0]['distance'] / 1000,
                answer['result']['routes'][0]['toll']]
        print("過路費查詢成功，第%d 行" % (i+1))
    except:
        info=['wrong', 'wrong', 'wrong']
        print("過路費查詢失敗，第%d 行" % (i+1))
    # finally:
    #     for j in range(8, 11):
    #         _ = origin_table.cell(column=j, row=i + 3, value=info[j - 8])
    #     data.save(r"ODdata.xlsx")
    #     print("第%d 行保存成功" % (i + 1))
    return info



if __name__=='__main__':
    start = time.clock()

    data = load_workbook(r"ODdata.xlsx")
    table = data['locationcode']
    nrows = table.max_row
    ncols = table.max_column

    origin_table = data['OD']
    origin_nrows = origin_table.max_row
    origin_ncols = origin_table.max_column
    go_outset = []
    go_destination = []
    for r in range(2,nrows+1):
        go_outset.append(table.cell(row=r,column=2).value) #生成去程出發(fā)地列表
    for r in range(2,nrows+1):
        go_destination.append(table.cell(row=r,column=5).value) #生成去程目的地列表

    go_outset_count=1
    go_destination_count=1
    go_outset_locationcode = []
    for i in go_outset:
        try:
            go_outset_locationcode.append(geocode(i)) #生成去程出發(fā)地經(jīng)緯度
            print("出發(fā)地經(jīng)緯度查詢計數(shù)%d"%go_outset_count)
            go_outset_count+=1
        except:
            go_outset_locationcode.append({'lat':'wrong','lng':'wrong'})

    go_destination_locationcode = []
    for i in go_destination:
        try:
            go_destination_locationcode.append(geocode(i))#生成去程目的地經(jīng)緯度
            print("目的地經(jīng)緯度查詢計數(shù)%d" % go_destination_count)
            go_destination_count+=1
        except:
            go_destination_locationcode.append({'lat':'wrong','lng':'wrong'})

    go_outset_locationcodelist = []
    go_destination_locationcodelist = []

    for i in range(len(go_outset_locationcode)):
        go_outset_locationcodelist.append(go_outset_locationcode[i].values())

    for i in range(len(go_destination_locationcode)):
        go_destination_locationcodelist.append(go_destination_locationcode[i].values())

    #將經(jīng)緯度和省份寫入 excel
    for i in range(2,nrows+1):
        for j in range(3,5):
            _ = table.cell(column=j, row=i, value=list(go_outset_locationcodelist[i-2])[j-3])
    for i in range(2,nrows+1):
        for j in range(6,8):
            _ = table.cell(column=j, row=i, value=list(go_destination_locationcodelist[i-2])[j-6])
    data.save(r"ODdata.xlsx")

    #開啟多進(jìn)程，獲取過路費
    for i in range(0,len(go_outset)):
        if list(go_outset_locationcodelist[i])[0]=='wrong':
            continue
        else:
            pool = multiprocessing.Pool(processes=5)
            pool.apply_async(getall,(i,list(go_outset_locationcodelist[i])[0], list(go_outset_locationcodelist[i])[1],list(go_destination_locationcodelist[i])[0], list(go_destination_locationcodelist[i])[1],),callback=save)
            # pool.apply(getall, (i, list(go_outset_locationcodelist[i])[0], list(go_outset_locationcodelist[i])[1],
            #                           list(go_destination_locationcodelist[i])[0],
            #                           list(go_destination_locationcodelist[i])[1],))
            # p = multiprocessing.Process(target=save, args=(i,list(go_outset_locationcodelist[i])[0], list(go_outset_locationcodelist[i])[1],
            #                 list(go_destination_locationcodelist[i])[0], list(go_destination_locationcodelist[i])[1],lock))
            # p.start()
            pool.close()
            pool.join()

    elapsed = (time.clock() - start)
    print("Time used:", elapsed)

回答

編輯回答

還吻

多線程應(yīng)該是這么操作的

#coding:utf-8
from multiprocessing.dummy import Pool #這里使用協(xié)程而不是線程,根據(jù)自己的情況選擇,協(xié)程可以開更多的thread
import time


def callback(result): # 回調(diào)
    print result
def dealer(name): # 處理函數(shù)
    time.sleep(3) # do some heavy job
    return name

pool = Pool(20) # 準(zhǔn)備一個20個thread的線程池
p = ["task1","task2","taskn"] # n個任務(wù)
for task in p:
    pool.apply_async(dealer,(task,),callback=callback)
pool.close()
pool.join()

關(guān)鍵在join方法,這個方法是用來阻塞進(jìn)程,保證多線程執(zhí)行完畢的,你每循環(huán)到一個任務(wù),就初始化一個線程池,并且阻塞到這個任務(wù)結(jié)束,其他的任務(wù)并沒有同時開始執(zhí)行,到下次循環(huán)才會開始另一個任務(wù).
Pool只應(yīng)該初始化一次,你的代碼里,在for循環(huán)中反復(fù)初始化Pool,并且調(diào)用join方法阻塞了程序,實際上你的代碼還是單線程在跑.
把我這個例子寫成你的風(fēng)格就是

#coding:utf-8
from multiprocessing.dummy import Pool #這里使用協(xié)程而不是線程,根據(jù)自己的情況選擇,協(xié)程可以開更多的thread
import time


def callback(result): # 回調(diào)
    print result
def dealer(name): # 處理函數(shù)
    time.sleep(3) # do some heavy job
    return name


p = ["task1","task2","taskn"] # n個任務(wù)
for task in p:
    pool = Pool(20) # 準(zhǔn)備一個20個thread的線程池
    pool.apply_async(dealer,(task,),callback=callback)
    pool.close()
    pool.join()

這兩段代碼都可以直接復(fù)制出來跑,你感受一下就明白了.

2018年9月21日 17:55