多进程使用multiprocessing模块,使用多进程的目的,是提高cpu的利用率。
Queue \ Pipe 可以实现进程间数据的传递,Manager 实现了进程间数据的共享,即多个进程可以修改同一份数据。
多进程使用Lock示例
Lock用来锁定进程间共享的资源,防止争夺。
from multiprocessing import Process, Lock
def f(l, i):
l.acquire()
print('hello world', i)
l.release()
if __name__ == '__main__':
lock = Lock()
ps = []
for num in range(10):
p = Process(target = f, args = (lock, num))
p.start()
ps .append(p)
for p in ps:
p.join()
多进程使用Pipe示例
多进程使用Pipe消息传送的形式共享数据。
import time
from multiprocessing import Process, Pipe
def f(conn):
while True:
cmd = conn.recv()
print("child process received:", cmd)
if cmd == "quit" or cmd == "exit":
break
time.sleep(1)
conn.send(["child process cmd complete:", cmd])
conn.close()
if __name__ == '__main__':
parent_conn, child_conn = Pipe()
p = Process(target = f, args = (child_conn,))
p.start()
while True:
cmd = input()
print("you input:", cmd)
parent_conn.send(cmd)
if cmd == "quit" or cmd == "exit":
break
print(parent_conn.recv())
p.join()
多进程使用Queue示例
from multiprocessing import Process, Queue
import threading
def f(qq):
print("in child:",qq.qsize())
qq.put([42, None, 'hello'])
if __name__ == '__main__':
q = Queue()
q.put("test123")
p = Process(target = f, args=(q,))
p.start()
p.join()
print(q.get_nowait())
print(q.get_nowait())
多进程使用示例Manager
from multiprocessing import Process, Manager
import os
def f(i, d, l):
d[i] = os.getpid()
l.append(os.getpid())
if __name__ == '__main__':
with Manager() as manager:
d = manager.dict()
l = manager.list(range(5))
p_list = []
for i in range(10):
p = Process(target=f, args=(i, d, l))
p.start()
p_list.append(p)
for res in p_list:
res.join()
print(d)
print(l)
进程池使用示例
from multiprocessing import Process, Pool
import os
import time
def Foo(i):
time.sleep(2)
print("in process",os.getpid())
return i + 100
def Bar(arg):
print('-->exec done:', arg, os.getpid())
if __name__ == '__main__':
pool = Pool(processes = 3)
print("main process:", os.getpid())
for i in range(10):
pool.apply_async(func = Foo, args = (i, ), callback = Bar)
pool.close()
pool.join()
多进程多线程综合使用示例
import multiprocessing
import time,threading, os
def thread_run(parent_id):
print("parent_id", parent_id, " thread:", threading.get_ident())
def run(name):
time.sleep(2)
print('hello', name)
t = threading.Thread(target = thread_run, args = (os.getpid(),))
t.start()
t.join()
if __name__ == '__main__':
for i in range(10):
p = multiprocessing.Process(target = run, args = ('Shixuan%s' %i,))
p.start()
p.join()