python3.6_多进程_multiprocessing.pool_concurrent.futures_ProcessPoolExecutor
转载注明来源: 本文链接 来自osnosn的博客，写于 2020-06-27.
多进程的多种写法，在大量任务的情况下，效率的对比。
import time
from multiprocessing.pool import Pool
from concurrent.futures import as_completed, ProcessPoolExecutor
NUMBERS = range(1, 60000)
K = 50
def f(x):
r = 0
for k in range(1, K+2):
r += x ** (1 / k**1.5)
return ['xx',r]
if __name__ == '__main__':
if 1:
print('-------------------\n no multiProcessing:')
start = time.time()
l = []
for nn in NUMBERS:
result=f(nn)
l.append(result)
print(len(l), l[0])
print('COST: {}'.format(time.time() - start))
if 1:
print('-------------------\n multiprocessing.pool.Pool:')
start = time.time()
l = []
pool = Pool(4)
for num, result in zip(NUMBERS, pool.map(f, NUMBERS)):
l.append(result)
print(len(l), l[0])
print('COST: {}'.format(time.time() - start))
if 1:
print('-------------------\n multiprocessing.pool.Pool, apply_async:')
start = time.time()
l = []
pool = Pool(4)
res=[]
for nn in NUMBERS:
res.append(pool.apply_async(f,(nn,)))
pool.close()
print('middle COST: {}'.format(time.time() - start))
pool.join()
for rr in res:
l.append(rr.get())
print(len(l), l[0])
print('COST: {}'.format(time.time() - start))
if 1:
print('-------------------\n multiprocessing.pool.Pool, apply_async,maxtasksperchild=1000 :')
start = time.time()
l = []
pool = Pool(4,maxtasksperchild=1000)
res=[]
for nn in NUMBERS:
res.append(pool.apply_async(f,(nn,)))
pool.close()
print('middle COST: {}'.format(time.time() - start))
pool.join()
for rr in res:
l.append(rr.get())
print(len(l), l[0])
print('COST: {}'.format(time.time() - start))
if 1:
print('-------------------\n ProcessPoolExecutor with chunksize,1/4:')
start = time.time()
l = []
with ProcessPoolExecutor(max_workers=4) as executor:
chunksize, extra = divmod(len(NUMBERS), executor._max_workers * 4)
print('chunksize',chunksize)
for num, result in zip(NUMBERS, executor.map(f, NUMBERS, chunksize=chunksize)):
l.append(result)
print(len(l), l[0])
print('COST: {}'.format(time.time() - start))
if 1:
print('-------------------\n ProcessPoolExecutor with chunksize,1/10:')
start = time.time()
l = []
with ProcessPoolExecutor(max_workers=4) as executor:
chunksize, extra = divmod(len(NUMBERS), executor._max_workers * 10)
print('chunksize',chunksize)
for num, result in zip(NUMBERS, executor.map(f, NUMBERS, chunksize=chunksize)):
l.append(result)
print(len(l), l[0])
print('COST: {}'.format(time.time() - start))
if 1:
print('-------------------\n ProcessPoolExecutor with chunksize,1/100:')
start = time.time()
l = []
with ProcessPoolExecutor(max_workers=4) as executor:
chunksize, extra = divmod(len(NUMBERS), executor._max_workers * 100)
print('chunksize',chunksize)
for num, result in zip(NUMBERS, executor.map(f, NUMBERS, chunksize=chunksize)):
l.append(result)
print(len(l), l[0])
print('COST: {}'.format(time.time() - start))
if 1:
print('-------------------\n ProcessPoolExecutor with chunksize,1/300:')
start = time.time()
l = []
with ProcessPoolExecutor(max_workers=4) as executor:
chunksize, extra = divmod(len(NUMBERS), executor._max_workers * 300)
print('chunksize',chunksize)
for num, result in zip(NUMBERS, executor.map(f, NUMBERS, chunksize=chunksize)):
l.append(result)
print(len(l), l[0])
print('COST: {}'.format(time.time() - start))
if 1:
print('-------------------\n ProcessPoolExecutor with chunksize,500:')
start = time.time()
l = []
with ProcessPoolExecutor(max_workers=4) as executor:
chunksize=500
print('chunksize',chunksize)
for num, result in zip(NUMBERS, executor.map(f, NUMBERS, chunksize=chunksize)):
l.append(result)
print(len(l), l[0])
print('COST: {}'.format(time.time() - start))
if 1:
print('-------------------\n ProcessPoolExecutor submit:')
start = time.time()
pool_res=[]
executor=ProcessPoolExecutor(max_workers=4)
for nn in NUMBERS:
res=executor.submit(f,nn)
pool_res.append(res)
print('middle COST: {}'.format(time.time() - start))
l = []
for p_res in as_completed(pool_res):
result=p_res.result()
l.append(result)
print(len(l), l[0])
print('COST: {}'.format(time.time() - start))
if 1:
print('-------------------\n ProcessPoolExecutor without chunksize:')
start = time.time()
l = []
with ProcessPoolExecutor(max_workers=4) as executor:
for num, result in zip(NUMBERS, executor.map(f, NUMBERS)):
l.append(result)
print(len(l), l[0])
print('COST: {}'.format(time.time() - start))
print('')
结果：
-------------------
no multiProcessing:
59999 ['xx', 51.0]
COST: 1.2773692607879639
-------------------
multiprocessing.pool.Pool:
59999 ['xx', 51.0]
COST: 0.4585001468658447
-------------------
multiprocessing.pool.Pool, apply_async:
middle COST: 1.492830514907837
59999 ['xx', 51.0]
COST: 4.116384267807007
-------------------
multiprocessing.pool.Pool, apply_async,maxtasksperchild=1000 :
middle COST: 2.0289459228515625
59999 ['xx', 51.0]
COST: 5.032078266143799
-------------------
ProcessPoolExecutor with chunksize,1/4:
chunksize 3749
59999 ['xx', 51.0]
COST: 0.4767882823944092
-------------------
ProcessPoolExecutor with chunksize,1/10:
chunksize 1499
59999 ['xx', 51.0]
COST: 0.5644888877868652
-------------------
ProcessPoolExecutor with chunksize,1/100:
chunksize 149
59999 ['xx', 51.0]
COST: 0.4668114185333252
-------------------
ProcessPoolExecutor with chunksize,1/300:
chunksize 49
59999 ['xx', 51.0]
COST: 0.673607587814331
-------------------
ProcessPoolExecutor with chunksize,500:
chunksize 500
59999 ['xx', 51.0]
COST: 0.45476365089416504
-------------------
ProcessPoolExecutor submit:
middle COST: 11.38172698020935
59999 ['xx', 16145.294670113708]
COST: 21.179430723190308
-------------------
ProcessPoolExecutor without chunksize:
59999 ['xx', 51.0]
COST: 20.61406421661377