browsermobproxy代理抓取UI页面接口HTTP请求

版权声明：本文为博主原创文章，未经博主允许不得转载。 https://blog.csdn.net/zhanghs11/article/details/80487030

H5 UI自动化，想抓取network的接口响应，第三方库browsermobproxy支持HTTP的抓取。文档说支持HTTPS，但实际一直报证书错误。实际也导入了证书。花费了1整天的时间研究，但是并木有解决。。。记录下这个问题吧~

#-*- coding = utf-8 -*-
import os, re
from browsermobproxy import Server
from selenium import webdriver
from uiBaseMethod import *


class Monitor(object):
    def __init__(self):
        # 代理的存放路径
        self.PROXY_PATH = ''.join([os.getcwd(), '\\dirver\\browsermob-proxy2.1\\bin\\browsermob-proxy.bat'])
        # chrome的存放路径
        self.CHROME_PATH = ''.join([os.getcwd(), '\\dirver\\chromedriver.exe'])
        self.CHROME_OPTIONS = {"profile.managed_default_content_settings.images":2}

    def initProxy(self):
        self.server = Server(self.PROXY_PATH)
        self.server.start()
        self.proxy = self.server.create_proxy()
        #self.proxy.blacklist(["http://.*/.*.css.*","http://.*/.*.jpg.*","http://.*/.*.png.*","http://.*/.*.gif.*"],200)

    def initChrome(self):
        # 设置ssl不生效，证书安装有问题，暂时没解决，无法抓https的接口
        # chrome_desired_capabilities = webdriver.DesiredCapabilities().FIREFOX
        # chrome_desired_capabilities['acceptSslCerts'] = True

        chrome_options = webdriver.ChromeOptions()
        print(self.proxy.port)
        chrome_options.add_argument('--proxy-server={host}:{port}'.format(host = "localhost", port = self.proxy.port))
        chrome_options.add_argument('--ignore-certificate-errors')
        chrome_options.add_experimental_option("prefs", self.CHROME_OPTIONS)
        self.driver = webdriver.Chrome(executable_path = self.CHROME_PATH, chrome_options = chrome_options)
        # self.driver = webdriver.Chrome(executable_path = self.CHROME_PATH, chrome_options = chrome_options, desired_capabilities = chrome_desired_capabilities)

    def createCapture(self, name = "monitor", options={'captureContent':True}):
        ''' 创建一个监控：captureContent = True'''
        self.proxy.new_har(name, options = options)

    def getResponseText(self, targetUrl):
        if self.proxy.har['log']['entries']:
            for loop_record in self.proxy.har['log']['entries']:
                try:
                    if re.fullmatch(targetUrl, loop_record["request"]['url']):
                        return loop_record["response"]['content']["text"]
                except Exception as err:
                    print(err)
                    continue
        return None


    def Start(self):
        try:
            self.initProxy()
            self.initChrome()
        except Exception as err:
            print(err)

    def Quit(self):
        self.driver.close()
        self.driver.quit()
        try:
            self.proxy.close()
            self.server.process.terminate()
            self.server.process.wait()
            self.server.process.kill()
        except OSError:
            pass

if __name__ == "__main__":
    monitor = Monitor()
    monitor.Start()
    monitor.createCapture()
    # 封装的登录模块
    login(monitor.driver, 'user_name', 'user_pwd')
    target_url = "http://lcts1.ssjlicai.com/userws/ws/login/v1/loginByPwd.*?"
    text = monitor.getResponseText(target_url)
    monitor.Quit()