python requests 库

python requests 库详解接口自动化及爬虫时会用到

摘自网络爬虫开发实战 -python3

reuqests 乱码
=======响应
r=requests.get("https://www.baidu.com")
r.encoding="utf-8"   # 转换编码格式
print(type(r))
print(r.status_code)
print(type(r.text))
print(r.text)
print(r.url)
print(r.cookies)
print(r.content)
print(r.history)

=========requests get请求 params ***
data={
    'name':'lan',
    'age':22
}
r=requests.get('http://httpbin.org/get',params=data)
print(r.text)

r.json()   # 得到一个字典的话直接调用json() 方法；如果返回结果不是json格式的话回报
json.decoder.JSONDecodeError 异常
type(r.json())

添加headers

headers={
   'user-agent':'xxxxxxxxxxxxxxxxxxxx'
}
r=requests.get(url,headers=headers)

=======requests post     请求

data={'name':'age':'22'}

r=requests.post(url,data=data)
print(r.text)


高级用法
=======1. 文件上传
files={'file':open('fav.ico','rb')}
r=requests.post(url,files=files)
print(r.text)

cookies ####
r=requests.get("http://www.baidu.com")
print(r.cookies)
for key,value in r.cookies.items():
   print(key+"="+value)

=======cookies 设置

cookies="xxxxxxxxxxxxxxxxxxxx"
jar=requests.cookies.RequestsCookieJar()
jar.set(key.value)

=======ssl 证书验证
response=requests.get("http://www.12306.cn",verify=False)

其中
忽略警告方式
from requests.packages import urllib3

urllib3.disable_warnings()   # 忽略警告
response=requests.get("http://www.12306.cn",verify=False)

通过捕获警告到日志的方式
import logging

logging.captureWarnings(True)

========代理设置
proxies={
   "http":"http://10.10.1.10:3128",
   "https":"https://10.10.1.10:6789"
}
response=requests.get("http://www.12306.cn",proxies=proxies)

=========超时设置
response=requests.get("http://www.12306.cn",timeout=1)   连接和读取
response=requests.get("http://www.12306.cn",timeout=(5,3))    连接和读取分别读取##### timeout=None 或者不加参数

=========身份验证
response=requests.get("http://www.12306.cn",auth=("username","password"))

=========OAuth 认证
pip install requests_oauthlib

from requests_oauthlib import OAuth1
auth=OAuth1("YOUR_APP_KEY","YOUR_APP_SECRET","USER_OAUTH_TOKEN","USER_OAUTH_TOKEN_SECRET")
response=requests.get("http://www.12306.cn",auth=auth)

=========Prepared Request

from requests import Request,Session

url={}
data={}
headers={}
s=Seesion()
req=Request("Post",url,data=data,headers=headers)
prepped=s.prepare_request(req)
r=s.send(prepped)
print(r.text)

题外
split()
u="www.doiido.com.cn"
>>> u1,u2,u3 = u.split('.',2)
>>> print u1
www
>>> print u2
doiido
>>> print u3
com.cn

>>> str="hello boy<[www.doiido.com]>byebye"
>>> print str.split("[")[1].split("]")[0]
www.doiido.com
>>> print str.split("[")[1].split("]")[0].split(".")
['www', 'doiido', 'com']

猜你喜欢