Java爬虫HTTPClient -- GET简单爬虫测试

java文件
不带参的写法：

package xxx.xxx.xxx;


import com.sun.javafx.webkit.CursorManagerImpl;
import com.sun.xml.internal.ws.policy.privateutil.PolicyUtils;
import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;

import java.io.IOException;


public class crawlerFirst {

    public static void main(String[] args)  {
        //1. 打开浏览器，创建HTTPClient对象
        CloseableHttpClient httpClient = HttpClients.createDefault();
        //2.输入网址，发起get请求创建HttpGet对象
        HttpGet httpGet = new HttpGet("http://www.itcast.cn");
        //3.按回车，发起请求，返回响应，使用HTTPClient对象发起请求
        CloseableHttpResponse response = null;
       try{
           response = httpClient.execute(httpGet);

           //4.解析响应，获取数据
           if( response.getStatusLine().getStatusCode() == 200){
               HttpEntity httpEntity = response.getEntity();
               String content = EntityUtils.toString(httpEntity, "utf8");
               System.out.println(content);
           }
       }catch(IOException e){
           e.printStackTrace();
       }finally {
           //关闭response
           try{
               response.close();
           }catch(IOException e){
               e.printStackTrace();
           }
            try{
                httpClient.close();
            }catch(IOException e){
                e.printStackTrace();
            }
       }


    }
}

运行结果：

带参数的写法：

用到了URIBuilder这个类

package xxx.xxx.xxx;


import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;

import java.io.IOException;
import java.net.URISyntaxException;


public class HttpGetParamTest {

    public static void main(String[] args) throws Exception {
        //1. 打开浏览器，创建HTTPClient对象
        CloseableHttpClient httpClient = HttpClients.createDefault();

//        设置请求地址是:http://yun.itheima.com/search?keys=Java
//        创建URIBuilder
        URIBuilder uriBuilder = new URIBuilder("http://yun.itheima.com/search?keys=Java");
        uriBuilder.setParameter("keys","Java");

        //2.输入网址，发起get请求创建HttpGet对象
        HttpGet httpGet = new HttpGet(uriBuilder.build());

        System.out.println("发送请求的信息："+httpGet);
        //3.按回车，发起请求，返回响应，使用HTTPClient对象发起请求
        CloseableHttpResponse response = null;
       try{
           response = httpClient.execute(httpGet);

           //4.解析响应，获取数据
           if( response.getStatusLine().getStatusCode() == 200){
               HttpEntity httpEntity = response.getEntity();
               String content = EntityUtils.toString(httpEntity, "utf8");
               System.out.println(content);
           }
       }catch(IOException e){
           e.printStackTrace();
       }finally {
           //关闭response
           try{
               response.close();
           }catch(IOException e){
               e.printStackTrace();
           }
            try{
                httpClient.close();
            }catch(IOException e){
                e.printStackTrace();
            }
       }


    }
}

效果截图：