python进阶--异步

3. 异步

3.1 asyncio

python3.4开始引入的标准库,内置了对异步io的支持
asyncio本身是一个消息循环
步骤:
    创建消息循环
    把协程导入
    关闭

import asyncio

@asyncio.coroutine
def hello():
    print("Hello world!")
    # 异步调用asyncio.sleep(1):
    print("Start......")
    r = yield from asyncio.sleep(3)
    print("Done....")
    print("Hello again!")

# 获取EventLoop:
loop = asyncio.get_event_loop()
# 执行coroutine
loop.run_until_complete(hello())
loop.close()

>Hello world!
 Start......
 Done....
 Hello again!

案例:得到多个网站

import asyncio

@asyncio.coroutine
def wget(host):
    print('wget %s...' % host)
    connect = asyncio.open_connection(host, 80)
    reader, writer = yield from connect
    header = 'GET / HTTP/1.0\r\nHost: %s\r\n\r\n' % host
    writer.write(header.encode('utf-8'))
    yield from writer.drain()
    while True:
        line = yield from reader.readline()
        if line == b'\r\n':
            break
    print('%s header > %s' % (host, line.decode('utf-8').rstrip()))
    # Ignore the body, close the socket
    writer.close()

loop = asyncio.get_event_loop()
tasks = [wget(host) for host in ['www.sina.com.cn', 'www.sohu.com', 'www.163.com']]
loop.run_until_complete(asyncio.wait(tasks))
loop.close()

>
wget www.163.com...
wget www.sina.com.cn...
wget www.sohu.com...
www.163.com header > HTTP/1.0 302 Moved Temporarily
www.163.com header > Server: Cdn Cache Server V2.0
www.163.com header > Date: Sun, 26 Aug 2018 13:44:32 GMT
www.163.com header > Content-Length: 0
www.163.com header > Location: http://www.163.com/special/0077jt/error_isp.html
www.163.com header > Connection: close
www.sina.com.cn header > HTTP/1.1 302 Moved Temporarily
www.sina.com.cn header > Server: nginx
www.sina.com.cn header > Date: Sun, 26 Aug 2018 13:44:32 GMT
www.sina.com.cn header > Content-Type: text/html
www.sina.com.cn header > Content-Length: 154
www.sina.com.cn header > Connection: close
www.sina.com.cn header > Location: https://www.sina.com.cn/
www.sina.com.cn header > X-Via-CDN: f=edge,s=ctc.wuhan.ha2ts4.30.nb.sinaedge.com,c=59.174.49.231;
www.sina.com.cn header > X-Via-Edge: 1535291072741e731ae3b7e84af3b477882f5
www.sohu.com header > HTTP/1.1 200 OK
www.sohu.com header > Content-Type: text/html;charset=UTF-8
www.sohu.com header > Connection: close
www.sohu.com header > Server: nginx
www.sohu.com header > Date: Sun, 26 Aug 2018 13:43:44 GMT
www.sohu.com header > Cache-Control: max-age=60
www.sohu.com header > X-From-Sohu: X-SRC-Cached
www.sohu.com header > Content-Encoding: gzip
www.sohu.com header > FSS-Cache: HIT from 9449353.16461715.11224052
www.sohu.com header > FSS-Proxy: Powered by 3813171.5189437.5587784

3.2 async and await

为了更好的表示异步io
python3.5 开始引入
让coroutine代码更简洁
使用上,可以简单进行替换:
    - 可以把 @asyncio.coroutine 替换成async
    - yield from 替换成await

import threading
import asyncio

#@asyncio.coroutine
async  def hello():
    print('Hello world! (%s)' % threading.currentThread())
    print('Start..... (%s)' % threading.currentThread())
    await asyncio.sleep(10)
    print('Done..... (%s)' % threading.currentThread())
    print('Hello again! (%s)' % threading.currentThread())

loop = asyncio.get_event_loop()
tasks = [hello(), hello()]
loop.run_until_complete(asyncio.wait(tasks))
loop.close()

>略

3.3 aiohttp

asyncio实现单线程并发IO,在客户端用处不大
在服务器端可以asyncio+coroutine配合,因为http是io操作
asyncio实现了TCP,UIDP,SSL等协议
aiohttp是给予asyncio实现的HTTP框架

import asyncio

from aiohttp import web

async def index(request):
    await asyncio.sleep(0.5)
    return web.Response(body=b'<h1>Index</h1>')

async def hello(request):
    await asyncio.sleep(0.5)
    text = '<h1>hello, %s!</h1>' % request.match_info['name']
    return web.Response(body=text.encode('utf-8'))

async def init(loop):
    app = web.Application(loop=loop)
    app.router.add_route('GET', '/', index)
    app.router.add_route('GET', '/hello/{name}', hello)
    srv = await loop.create_server(app.make_handler(), '127.0.0.1', 8000)
    print('Server started at http://127.0.0.1:8000...')
    return srv

loop = asyncio.get_event_loop()
loop.run_until_complete(init(loop))
loop.run_forever()

3.4 concurrent.futures

类似其他语言的线程池的概念,此模块利用multiprocessiong实现真正的平行计算
核心原理是：concurrent.futures会以子进程的形式，平行的运行多个python解释器，从而令python程序可以利用多核CPU来提升执行速度。
- 由于子进程与主解释器相分离，所以他们的全局解释器锁也是相互独立的。每个子进程都能够完整的使用一个CPU内核。

concurrent.futures.Executor
    - ThreadPoolExecutor
    - ProcessPoolExecutor
submit(fn, args, kwargs)
    - fn:异步执行的函数
    - args,kwargs:参数

# 官方死锁案例
import time
def wait_on_b():
    time.sleep(5)
    print(b.result())  #b不会完成，他一直在等待a的return结果
    return 5

def wait_on_a():
    time.sleep(5)
    print(a.result())  #同理a也不会完成，他也是在等待b的结果
    return 6


executor = ThreadPoolExecutor(max_workers=2)
a = executor.submit(wait_on_b)
b = executor.submit(wait_on_a)

map(fn, \*iterables, timeout=None)
- 跟map函数类似
- 函数需要异步执行
- timeout: 超时时间
- 起执行结果是list,数据需要从list中取出来
- submit和map根据需要选一个即可

import time,re
import os,datetime
from concurrent import futures

data = ['1','2']

def wait_on(argument):
   print(argument)
   time.sleep(2)
   return "ok"

ex = futures.ThreadPoolExecutor(max_workers=2)
for i in ex.map(wait_on,data):
   print(i)

 Future
- 未来需要完成的任务
- future 实例由Excutor.submit创建

from concurrent.futures import ThreadPoolExecutor as Pool
from concurrent.futures import as_completed
import requests

URLS = ['http://qq.com', 'http://sina.com', 'http://www.baidu.com', ]

def task(url, timeout=10):
    return requests.get(url, timeout=timeout)

with Pool(max_workers=3) as executor:
    future_tasks = [executor.submit(task, url) for url in URLS]

    for f in future_tasks:
        if f.running():
            print('%s is running' % str(f))

    for f in as_completed(future_tasks):
        try:
            ret = f.done()
            if ret:
                f_ret = f.result()
                print('%s, done, result: %s, %s' % (str(f), f_ret.url, len(f_ret.content)))
        except Exception as e:
            f.cancel()
            print(str(e))