- 最近在做得工作和遇到的坑
- 阿里云OSS 图片上传服务
- 对接第三方网站的API 调试,使用httpclient请求对方的服务,对返回json进行解析,读取数据
划重点,
- https请求要用jdk1.8 ,不要使用jdk1.7 ,会报socket refuse,,或者socket connection close错误 2.
- 请求完毕后,一定要释放连接 response.close();
- 关于跨域请求 ,服务器端可以在代码中硬编码,或者使用过滤器,或者在nginx web服务器中配置,或者在浏览器请求方解决
this.getResponse().setHeader("Access-Control-Allow-Origin", "*");
this.getResponse().setHeader("Access-Control-Allow-Methods", "POST, GET, OPTIONS, DELETE");
this.getResponse().setHeader("Access-Control-Max-Age", "3600");
// this.getResponse().setHeader("Access-Control-Allow-Headers", "x-requested-with");
引入lib包 httpclient-4.4.1.jar
封装通用请求头
public static String reqPack(String command) {
Map<String, String> params = Collections.singletonMap("query", command);
Map<String, String> headers = new HashMap<String, String>();
headers.put("key", APPKEY);
headers.put("altitoken", getToken());
headers.put("Content-type", "application/x-www-form-urlencoded");
headers.put("User-Agent", "Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; DigExt)");
//传入请求地址,参数,请求头
return DataCrawlingHelper.sendPost(ALTIZURE_API_URL, params, headers);
}
- 封装请求类
package com.momoda.client;
import org.apache.http.HttpEntity;
import org.apache.http.NameValuePair;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
/**
* A站数据抓取类
*/
public class DataCrawlingHelper {
public static String sendPost(String requestUrl, Map<String, String> params, Map<String, String> headers) {
String body = "";
//创建httpclient对象
CloseableHttpClient client = HttpClients.createDefault();
//创建post方式请求对象
HttpPost httpPost = new HttpPost(requestUrl);
//装填参数
List<NameValuePair> nvps = new ArrayList<NameValuePair>();
if (params != null) {
for (Entry<String, String> entry : params.entrySet()) {
nvps.add(new BasicNameValuePair(entry.getKey(), entry.getValue()));
}
}
try {
//设置参数到请求对象中
httpPost.setEntity(new UrlEncodedFormEntity(nvps, "utf-8"));
//设置header信息
headers.forEach((key, val) -> {
httpPost.setHeader(key, val);
});
//执行请求操作,并拿到结果(同步阻塞)
CloseableHttpResponse response = client.execute(httpPost);
//获取结果实体
HttpEntity entity = response.getEntity();
if (entity != null) {
//按指定编码转换结果实体为String类型
body = EntityUtils.toString(entity, "utf-8");
}
EntityUtils.consume(entity);
//释放链接
response.close();
} catch (Exception e) {
e.printStackTrace();
return null;
}
return body;
}
}
操作返回数据
- 一般网站的返回数据都是通用的JSON格式,所以我们需要对json进行操作,如果我们抓取的事第三方网站的数据,一般是没有对应的实体类的,所以我们可以利用fastjson-1.1.27.jar 这个lib包操作我们的返回数据,
/**
* 检查项目状态
*/
public void checkProjectState() {
String projectId = this.getPara("projectId");
String projectInfoBack = reqPack(String.format(COMMAND_QUERY_PROJECT, projectId));
JSONObject jsonObj2 = JSONObject.parseObject(projectInfoBack);
//根据json的类型,使用对应的类型接受返回值
JSONArray edges = jsonObj2.getJSONObject("data").getJSONObject("project").getJSONObject("tasks").getJSONArray("edges");
String taskState = jsonObj2.getJSONObject("data").getJSONObject("project").getString("taskState"); //Unmodified Failed Done
if ("Done".equals(taskState)) {
renderJson(taskState);
} else {
renderJson(edges);
}
}
利用自己封装的json解析类
- 抽象类 ,
package com.momoda.client.altizure;
import com.momoda.common.Const;
import java.lang.reflect.ParameterizedType;
import java.lang.reflect.Type;
public abstract class AltizureRestltParseSuper<T> {
private T data = null;
public AltizureRestltParseSuper(String json) {
Class<T> entityClass = null;
Type t = getClass().getGenericSuperclass();
if (t instanceof ParameterizedType) {
Type[] p = ((ParameterizedType) t).getActualTypeArguments();
entityClass = (Class<T>) p[0];
}
this.data = Const.gson.fromJson(json, entityClass);
}
public T data() {
return this.data;
}
}
- 继承类
public class AltizureRestltParse extends AltizureRestltParseSuper<Map> {
public AltizureRestltParse(String json) {
super(json);
}
/**
* @Description :1. 选择最近的 bucket
* @Author : xuyangyang
* @Date : 14:26 2018/6/1/001
*/
public String getGeoIPInfo() {
Map data = new HashMap();
data = (Map) data().get("data");
Map getGeoIPInfo = (Map) data.get("getGeoIPInfo");
System.out.println(getGeoIPInfo.toString());
ArrayList nearestBuckets = (ArrayList) getGeoIPInfo.get("nearestBuckets");
System.out.println(nearestBuckets);
Map bucket = (Map) nearestBuckets.get(0);
System.out.println(bucket);
System.out.println((String) bucket.get("bucket"));
return (String) bucket.get("bucket");
}
}
- 方法调用 ,new 继承类,传入json字符串,直接使用get()方法,获取key值,但是注意要使用正确的类型转换
public String getGeoIPInfo() {
Map<String, String> map = new HashMap<>();
map.put("query", "query{getGeoIPInfo{nearestBuckets {\n" +
" bucket\n" +
" display\n" +
" cloud\n" +
" distance\n" +
"}}}");
String body = null;
String bucket = null;
try {
body = HttpClientUtil.sendPost(map);
AltizureRestltParse jsonParse = new AltizureRestltParse(body);
bucket = jsonParse.getGeoIPInfo();
} catch (IOException e) {
e.printStackTrace();
}
return bucket;
}