python多线程，控制并发数

版权声明：本文为博主原创文章，未经博主允许不得转载。 https://blog.csdn.net/ZWX2445205419/article/details/88423218

模拟服务端

from flask import Flask
from flask import request
import time
import json

app = Flask(__name__)
@app.route('/test/', methods=['GET', 'POST'])
def test():
    return_data = {}
    try:
        data = json.loads(request.get_data())
        user = data['user']
        info = data['info']
        r_user, r_info = async_func(user, info)
        return_data['user'] = r_user
        return_data['info'] = r_info
    except Exception as e:
        return_data['error_num'] = str(1)
        return_data['error_msg'] = str(e)
    print return_data
    return json.dumps(return_data)


def async_func(user, info):
    time.sleep(1)
    return user, info


if __name__ == '__main__':
    app.run(host='0.0.0.0', port=4000, debug=True)

模拟客户端

#! -*- coding:utf-8 -*-
import Queue
import threading
import time
import urllib2
import sys
import json


url = 'http://0.0.0.0:4000/test/'
req = urllib2.Request(url, headers={'Content-Type': 'application/json'})
lock = threading.Lock()

# class ThreadNum(threading.Thread):
#     def __init__(self, queue):
#         threading.Thread.__init__(self)
#         self.queue = queue
#
#     def run(self):
#         while True:
#             # 消费者端，从队列中获取num
#             num = self.queue.get()
#             print("Retrieved", num)
#             time.sleep(1)
#             # 在完成这项工作之后，使用 queue.task_done() 函数向任务已经完成的队列发送一个信号
#             self.queue.task_done()
#         print "Consumer Finished"
#
#
# def main():
#     # 产生一个 threads pool, 并把消息传递给thread函数进行处理，这里开启10个并发
#     for i in range(5):
#         t = ThreadNum(queue)
#         t.setDaemon(True)
#         t.start()
#
#     # 往队列中填数据
#     for num in range(100):
#         queue.put(num)
#         # wait on the queue until everything has been processed
#
#     queue.join()


class Client(threading.Thread):
    def __init__(self, queue):
        super(Client, self).__init__()
        self.queue = queue

    def run(self):
        while True:
            img_url = self.queue.get()
            if img_url is None:
                break

            post_data = json.dumps({
                'user': 'test',
                'info': 'test info',
            })
            try:
                res = urllib2.urlopen(req, post_data)
                result = json.loads(res.read())
                lock.acquire()
                print result
                lock.release()
            except Exception as e:
                print 'Http Error: {}'.format(e)
            finally:
                self.queue.task_done()


def main():
    filename = sys.argv[1]
    thread_num = 100
    queue = Queue.Queue()

    count = 0
    stime = time.time()
    for i in range(thread_num):
        t = Client(queue)
        t.setDaemon(True)
        t.start()

    for index, line in enumerate(open(filename), 1):
        _, _, _, links_str, _ = line.strip('\n').split('\t')
        for link in links_str.split(','):
            queue.put(link)
            count += 1
            if count > 1000:
                break
        if count > 1000:
            break
    queue.join()
    etime = time.time()
    print 'All subprocess Done! use {} seconds.'.format(etime - stime)
    for i in range(10):
        queue.put(None)


if __name__ == '__main__':
    main()

使用Semaphore控制线程的数量

https://blog.csdn.net/qq_38065133/article/details/82529008