htmlunit获取js与ajax加载的网页内容

htmlunit 开源的java页面分析工具，读取页面后，使用htmlunit分析页面内容。模拟浏览器的运行，是一个没有界面的浏览器，运行迅速。

采用Rhinojs引擎。模拟js运行

引入依赖

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  <modelVersion>4.0.0</modelVersion>
  <groupId>com.open</groupId>
  <artifactId>HtmlUnitTest</artifactId>
  <version>0.0.1-SNAPSHOT</version>
  
  <dependencies>
  	
  	<dependency>
	    <groupId>net.sourceforge.htmlunit</groupId>
	    <artifactId>htmlunit</artifactId>
	    <version>2.26</version>
	</dependency>
	
	<dependency>
		<groupId>org.apache.httpcomponents</groupId>
		<artifactId>httpclient</artifactId>
		<version>4.5.2</version>
	</dependency>
  	
  	
  </dependencies>
</project>

import java.io.IOException;
import java.net.MalformedURLException;

import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.HtmlPage;

public class HtmlUnitTest {

	public static void main(String[] args) {
		WebClient webClient=new WebClient(); // 实例化Web客户端
		try {
			HtmlPage page=webClient.getPage("http://www.baidu.com"); // 解析获取页面
			System.out.println("网页html:"+page.asXml());
			System.out.println("==================================");
			System.out.println("网页文本："+page.asText());
		} catch (FailingHttpStatusCodeException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (MalformedURLException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}finally{
			webClient.close(); // 关闭客户端，释放内存
		}
	}
}

htmlunit模拟浏览器

WebClient使用代理IP

setCssEnabled 取消支持提升解析速度

getHtmlElementById 根据id获取指定div元素

getElementsByTagName 根据标签名字获取a标签

getByXPath 根据dom获取元素，//任意路径下获取div 下 ul li的第二个 超链接的名称

import java.io.IOException;
import java.net.MalformedURLException;
import java.util.List;

import com.gargoylesoftware.htmlunit.BrowserVersion;
import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.DomElement;
import com.gargoylesoftware.htmlunit.html.DomNodeList;
import com.gargoylesoftware.htmlunit.html.HtmlDivision;
import com.gargoylesoftware.htmlunit.html.HtmlListItem;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import com.gargoylesoftware.htmlunit.html.HtmlSpan;

public class HtmlUnit {

	public static void main(String[] args) {
		WebClient webClient=new WebClient(BrowserVersion.FIREFOX_52,"1.61.148.187",25394); // 实例化Web客户端端
		webClient.getOptions().setCssEnabled(false); // 取消css支持
		webClient.getOptions().setJavaScriptEnabled(false); // 取消javascript支持
		try {
			HtmlPage page=webClient.getPage("http://www.baidu.com"); // 解析获取页面
                        Thread.sleep(10000); // 休息10秒 等待htmlunit执行Js
			HtmlDivision div=page.getHtmlElementById("navMenu"); // 查找指定id的html dom元素
			System.out.println(div.asXml());
			System.out.println("==============");
                 HtmlForm form=page.getFormByName("myform"); // 获取搜索Form
			HtmlTextInput textField=form.getInputByName("q"); // 获取查询文本框
			HtmlSubmitInput button=form.getInputByName("submitButton"); // 获取提交按钮
			textField.setValueAttribute("java"); // 文本框“填入”数据
			HtmlPage resultPage=button.click(); // 模拟点击 获取查询结果页面
			System.out.println(resultPage.asXml());
                  System.out.println("==============");
			DomNodeList<DomElement>  aList=page.getElementsByTagName("a"); // 根据tag名称查询所有tag
			for(int i=0;i<aList.getLength();i++){
				DomElement a=aList.get(i);
				System.out.println(a.asXml());
			}
			System.out.println("==============");
			List<HtmlSpan> spanList=page.getByXPath("//div[@id='navMenu']/ul/li[2]/a/span");
			System.out.println(spanList.get(0).asText());
		} catch (FailingHttpStatusCodeException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (MalformedURLException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}finally{
			webClient.close(); // 关闭客户端，释放内存
		}
	}
}