python新手，寫了一個計算文件md5的程序，能跑通。然后嘗試改造成多線程時遇到疑惑。兩核心的機器，開雙線程時計算消耗的時間和不使用多線程一致，不知是因為什么問題導致。測試環(huán)境下一共44個文件，每個文件200MB-400MB不等，總大小12.1GB，兩段代碼的運行時間都是42秒左右。
后面又嘗試了使用多進程的方式，依然處理時間沒有改善

未使用多線程代碼：

#!/usr/bin/python3
import os, hashlib, binascii, pymysql, time, json, datetime

def listFiles(dir):
    paths = []
    for root,dirs,files in os.walk(dir):
        for file in files:
            paths.append(os.path.join(root,file))
            
    return paths
    
def calcMD5(filePath, block_size=2**20):
    md5 = hashlib.md5()
    f = open(filePath, 'rb')
    while True:
        data = f.read(block_size)
        if not data:
            break
        md5.update(data)
    f.close()
    return md5.hexdigest()
        
files = listFiles('/data/S01')

result = []

startTime = datetime.datetime.now()

for i in files:
    fileMD5 = calcMD5(i)
    result.append(fileMD5)

print(result)

endTime = datetime.datetime.now()
timeDiff = endTime - startTime
timeDiffSeconds = timeDiff.seconds
print('總費時{0}分鐘{1}秒'.format(int(timeDiffSeconds/60), int(timeDiffSeconds%60)))

使用多線程代碼：

 #!/usr/bin/python3
    import os, hashlib, binascii, pymysql, time, json, datetime, threading, queue
    
    def listFiles(dir):
        paths = []
        for root,dirs,files in os.walk(dir):
            for file in files:
                paths.append(os.path.join(root,file))
                
        return paths
        
            
    class threadMD5(threading.Thread):
        def __init__(self, queue):
            threading.Thread.__init__(self)
            self.queue = queue
        
        def run(self):
            while True:
                try:
                    filePath = self.queue.get(block=False)
                except Exception as e:
                    print('thread end')
                    break
                fileMD5 = calcMD5(filePath)
                
                self.queue.task_done()
    
    def calcMD5(filePath, block_size=2**20):
        md5 = hashlib.md5()
        f = open(filePath, 'rb')
        while True:
            data = f.read(block_size)
            if not data:
                break
            md5.update(data)
        f.close()
        return md5.hexdigest()
        
    startTime = datetime.datetime.now()
    files = listFiles('/data/S01')
    
    result = []
    
    #多線程
    queue = queue.Queue()
    for i in files:
        queue.put(i, block=False)
    
    threads = []
    
    for i in range(2):
        t = threadMD5(queue)
        t.setDaemon(True)
        t.start()
        threads.append(t)
    
    for i in threads:
        i.join()
    
    print(result)
    
    endTime = datetime.datetime.now()
    timeDiff = endTime - startTime
    timeDiffSeconds = timeDiff.seconds
    print('總費時{0}分鐘{1}秒'.format(int(timeDiffSeconds/60), int(timeDiffSeconds%60)))
    
多進程代碼：

import os, hashlib, time, datetime
import multiprocessing as mp

results = []

def listFiles(dir):
    paths = []
    for root,dirs,files in os.walk(dir):
        for file in files:
            paths.append(os.path.join(root,file))
            
    return paths

def calcMD5(filePath, block_size=2**20):
    md5 = hashlib.md5()
    f = open(filePath, 'rb')
    while True:
        data = f.read(block_size)
        if not data:
            break
        md5.update(data)
    f.close()
    return md5.hexdigest()

def collect_results(result):
    results.extend(result)

if __name__ == "__main__":
    p = mp.Pool(processes=2)
    files = listFiles('/data/S01')
    startTime = datetime.datetime.now()
    for f in files:
            p.apply_async(calcMD5, args=(f, ), callback=collect_results)
    p.close()
    p.join()
    print(results)
    
    endTime = datetime.datetime.now()
    timeDiff = endTime - startTime
    timeDiffSeconds = timeDiff.seconds
    print('總費時{0}分鐘{1}秒'.format(int(timeDiffSeconds/60), int(timeDiffSeconds%60)))