queue
ipc mechanism: Process Communications
Pipeline: pipe-based shared memory space
Queue: pipe + lock queue
from multiprocessing import Process,Queue
### 案例一
q = Queue()
q.put('hyc')
q.put([1,2,4])
q.put(2)
print(q.get())
print(q.get())
print(q.get())
# q.put(5)
# q.put(5)
print(q.get()) # 默认就会一直等着拿值
At this point the program to run will block here
from multiprocessing import Process,Queue
## 案例2
q = Queue(4)
q.put('我妻由乃')
q.put([1,2,4])
q.put('我妻善逸')
q.put(2)
q.put('乔碧萝') #队列满了的情况再放值,会阻塞
Is the same reason, have set the four values, when placed in the fifth value, it will be blocked
from multiprocessing import Process,Queue
q = Queue(3)
q.put('zhao',block=True,timeout=2) #
q.put('zhao',block=True,timeout=2) #
q.put('zhao',block=True,timeout=2) #
q.put('zhao',block=True,timeout=5) # put里的 block=True 如果满了会等待,timeout最多等待n s,如果ns还是队列还是满的就报错了
After waiting 5 seconds will complain
from multiprocessing import Process,Queue
q = Queue()
q.put('yyyy')
q.get()
q.get(block=True,timeout=5) # block=True 阻塞等待,timeout最多等5s, 剩下同上
When the value is the same reason
q = Queue(3)
q.put('qwe')
q.put('qwe')
q.put('qwe')
q.put('qwe',block=False) # 对于put来说block=False 如果队列满了就直接报错
q = Queue(3)
q.put('qwe')
q.get()
q.get(block=False)
# block = Flase 拿不到不阻塞,直接报错
When the block = Flase time, timeout just does not make sense, because it will direct error
q = Queue(1)
q.put('123')
# q.get()
q.put_nowait('666') # block = False
q.get_nowait() # block = False
Will direct error and in fact the same block = False, not blocking, direct error when there is a problem
Producer and consumer model
Manufacturer: production data task
Consumers: the task of processing data
The relationship between producers and consumers: producers - queue (pots) -> Consumers
Producers can not stop production, to reach their maximum productivity, consumers can keep spending, also reached their maximum consumption efficiency.
Producer-consumer model greatly improves the efficiency of production and consumer spending Producer s efficiency.
from multiprocessing import Process, Queue
def producer(q,name,food):
'''生产者'''
for i in range(10):
print(f'{name}生产了{food}{i}')
res = f'{food}{i}'
q.put(res)
q.put(None)
def consumer(q,name):
'''消费者'''
while True:
res = q.get(timeout=5)
if res is None:break
print(f'{name}吃了{res}')
if __name__ == '__main__':
q = Queue()
p1 = Process(target=producer,args=(q,'rocky','包子'))
c1 = Process(target=consumer,args=(q,'成哥'))
p1.start()
c1.start()
This is the simplest model of producers and consumers, and his role is that each producer will produce goods, when the producers of finished production of 10 buns, out of the for loop, and finally into a None. Then when the consumer receives None, it will be break out.
In a producer to a consumer when this method is feasible. But when multiple producers time to a consumer, one of the first producers of finished food production and put None, but the producer has not produced the second place finish, then the consumer has already received the first producer of None. In this case there will be a problem, so we can change the thinking about, put out to None
from multiprocessing import Process,Queue
import time,random
def producer(q,name,food):
'''生产者'''
for i in range(3):
print(f'{name}生产了{food}{i}')
time.sleep(random.randint(1, 3))
res = f'{food}{i}'
q.put(res)
# q.put(None)
def consumer(q,name):
'''消费者'''
while True:
res = q.get(timeout=5)
if res is None:break
time.sleep(random.randint(1,3))
print(f'{name}吃了{res}')
if __name__ == '__main__':
q = Queue()
p1 = Process(target=producer,args=(q,'rocky','包子'))
p2 = Process(target=producer,args=(q,'mac','韭菜'))
p3 = Process(target=producer,args=(q,'nick','蒜泥'))
c1 = Process(target=consumer,args=(q,'成哥'))
c2 = Process(target=consumer,args=(q,'浩南哥'))
p1.start()
p2.start()
p3.start()
c1.start()
c2.start()
p1.join()
p2.join()
p3.join() # 生产者生产完毕
q.put(None)# 几个消费者put几次
q.put(None)
But there will be new problems with this method, the method used is because consumers while True, he will not stop. So we have to use the daemon learned before:
from multiprocessing import Process,Queue,JoinableQueue
import time,random
def producer(q,name,food):
'''生产者'''
for i in range(3):
print(f'{name}生产了{food}{i}')
time.sleep(random.randint(1, 3))
res = f'{food}{i}'
q.put(res)
# q.put(None)
def consumer(q,name):
'''消费者'''
while True:
res = q.get()
# if res is None:break
time.sleep(random.randint(1,3))
print(f'{name}吃了{res}')
q.task_done() #
if __name__ == '__main__':
q = JoinableQueue()
p1 = Process(target=producer,args=(q,'rocky','包子'))
p2 = Process(target=producer,args=(q,'mac','韭菜'))
p3 = Process(target=producer,args=(q,'nick','蒜泥'))
c1 = Process(target=consumer,args=(q,'成哥'))
c2 = Process(target=consumer,args=(q,'浩南哥'))
p1.start()
p2.start()
p3.start()
c1.daemon = True
c2.daemon = True
c1.start()
c2.start()
p1.join()
p2.join()
p3.join() # 生产者生产完毕
# q.put(None)# 几个消费者put几次
# q.put(None)
q.join() # 分析
# 生产者生产完毕--这是主进程最后一行代码结束--q.join()消费者已经取干净了,没有存在的意义了.
#这是主进程最后一行代码结束,消费者已经取干净了,没有存在的意义了.守护进程的概念.
joinableQueue
from multiprocessing import Process,Queue,JoinableQueue
q = JoinableQueue()
q.put('zhao') # 放队列里一个任务
q.put('qian')
print(q.get())
q.task_done() # 完成了一次任务
print(q.get())
q.task_done() # 完成了一次任务
q.join() #计数器不为0的时候 阻塞等待计数器为0后通过
# 想象成一个计数器 :put +1 task_done -1