使用爬虫

其他 2019-01-11 22:26:17 阅读次数: 0 
     package com.bing.spider;
import java.io.IOException;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.bing.download.FileDownload;
import com.bing.html.HtmlManage;
import com.bing.http.HttpGetConnect;

import net.sf.json.JSONObject;

public class SpiderKugou {

 public static String filePath = "F:/music/";
 public static String mp3 = "https://wwwapi.kugou.com/yy/index.php?r=play/getdata&callback=jQuery191027067069941080546_1546235744250&"
     + "hash=HASH&album_id=0&_=TIME";
 
 public static String LINK = "https://www.kugou.com/yy/rank/home/PAGE-8888.html?from=rank";
 //"https://www.kugou.com/yy/rank/home/PAGE-23784.html?from=rank";
 
 
 public static void main(String[] args) throws IOException {
   
   for(int i = 1 ; i < 23 ; i++){
     String url = LINK.replace("PAGE", i + "");
     getTitle(url);
     //download("https://www.kugou.com/song/mfy6je5.html");
   }
 }
 
 public static String getTitle(String url) throws IOException{
   HttpGetConnect connect = new HttpGetConnect();
   String content = connect.connect(url, "utf-8");
   HtmlManage html = new HtmlManage();
   Document doc = html.manage(content);
   Element ele = doc.getElementsByClass("pc_temp_songlist").get(0);
   Elements eles = ele.getElementsByTag("li");
   for(int i = 0 ; i < eles.size() ; i++){
     Element item = eles.get(i);
     String title = item.attr("title").trim();
     String link = item.getElementsByTag("a").first().attr("href");
     
     download(link,title);
   }
   return null;
 }
 
 public static String download(String url,String name) throws IOException{
   String hash = "";
   HttpGetConnect connect = new HttpGetConnect();
   String content = connect.connect(url, "utf-8");
   HtmlManage html = new HtmlManage();
   
   String regEx = "\"hash\":\"[0-9A-Z]+\"";
   // 编译正则表达式
   Pattern pattern = Pattern.compile(regEx);
   Matcher matcher = pattern.matcher(content);
   if (matcher.find()) {
     hash = matcher.group();
     hash = hash.replace("\"hash\":\"", "");
     hash = hash.replace("\"", "");
   }
   
   String item = mp3.replace("HASH", hash);
   item = item.replace("TIME", System.currentTimeMillis() + "");

   System.out.println(item);
   String mp = connect.connect(item, "utf-8");
   
   mp = mp.substring(mp.indexOf("(") + 1, mp.length() - 3);
   
   JSONObject json = JSONObject.fromObject(mp);
   String playUrl = json.getJSONObject("data").getString("play_url");
   

   System.out.print(playUrl + "  ==  ");
   FileDownload down = new FileDownload();
   down.download(playUrl, filePath + name + ".mp3");
   
   System.out.println(name + "下载完成");
   return playUrl;
 }

}

HttpGetConnect.java

    package com.bing.http;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.security.NoSuchAlgorithmException;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;

import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManager;
import javax.net.ssl.X509TrustManager;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.http.HttpEntity;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpClient;
import org.apache.http.client.ResponseHandler;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.conn.ClientConnectionManager;
import org.apache.http.conn.scheme.Scheme;
import org.apache.http.conn.scheme.SchemeRegistry;
import org.apache.http.conn.ssl.SSLSocketFactory;
import org.apache.http.impl.client.BasicResponseHandler;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.BasicHttpClientConnectionManager;
import org.apache.http.params.HttpParams; 
/**
* @说明：
* @author: gaoll
* @CreateTime:2014-11-13
* @ModifyTime:2014-11-13
*/
public class HttpGetConnect {
 
 /**
  *  获取html内容
  * @param url
  * @param charsetName  UTF-8、GB2312
  * @return
  * @throws IOException
  */
 public static String connect(String url,String charsetName) throws IOException{
   BasicHttpClientConnectionManager connManager = new BasicHttpClientConnectionManager();
   
   CloseableHttpClient httpclient = HttpClients.custom()
             .setConnectionManager(connManager)
             .build();
   String content = "";
   
   try{
     HttpGet httpget = new HttpGet(url);
     
     RequestConfig requestConfig = RequestConfig.custom()
                 .setSocketTimeout(5000)
                 .setConnectTimeout(50000)
                 .setConnectionRequestTimeout(50000)
                 .build();
         httpget.setConfig(requestConfig);
         httpget.setHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
         httpget.setHeader("Accept-Encoding", "gzip,deflate,sdch");
         httpget.setHeader("Accept-Language", "zh-CN,zh;q=0.8");
         httpget.setHeader("Connection", "keep-alive");
         httpget.setHeader("Upgrade-Insecure-Requests", "1");
         httpget.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36");
         //httpget.setHeader("Hosts", "www.oschina.net");
         httpget.setHeader("cache-control", "max-age=0");  
       
     CloseableHttpResponse response = httpclient.execute(httpget);
     
     int status = response.getStatusLine().getStatusCode();
           if (status >= 200 && status < 300) {
             
               HttpEntity entity = response.getEntity();
               InputStream instream = entity.getContent();
               BufferedReader br = new BufferedReader(new InputStreamReader(instream,charsetName));
       StringBuffer sbf = new StringBuffer();
       String line = null;
       while ((line = br.readLine()) != null){
         sbf.append(line + "\n");
       }

       br.close();
       content = sbf.toString();
           } else {
               content = "";
           }
     
   }catch(Exception e){
     e.printStackTrace();
   }finally{
     httpclient.close();
   }
   //log.info("content is " + content);
   return content;
 }
 private static Log log = LogFactory.getLog(HttpGetConnect.class);
}

HtmlManage.java

     package com.bing.html;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.bing.http.HttpGetConnect;

/**
* @说明：
* @author: gaoll
* @CreateTime:2014-11-13
* @ModifyTime:2014-11-13
*/
public class HtmlManage {
 
 public Document manage(String html){
   Document doc = Jsoup.parse(html);
   return doc;
 }
 
 public Document manageDirect(String url) throws IOException{
   Document doc = Jsoup.connect( url ).get();
   return doc;
 }
 
 public List<String> manageHtmlTag(Document doc,String tag ){
   List<String> list = new ArrayList<String>();
   
   Elements elements = doc.getElementsByTag(tag);
   for(int i = 0; i < elements.size() ; i++){
     String str = elements.get(i).html();
     list.add(str);
   }
   return list;
 }
 
 public List<String> manageHtmlClass(Document doc,String clas ){
   List<String> list = new ArrayList<String>();
   
   Elements elements = doc.getElementsByClass(clas);
   for(int i = 0; i < elements.size() ; i++){
     String str = elements.get(i).html();
     list.add(str);
   }
   return list;
 }
 
 public List<String> manageHtmlKey(Document doc,String key,String value ){
   List<String> list = new ArrayList<String>();
   
   Elements elements = doc.getElementsByAttributeValue(key, value);
   for(int i = 0; i < elements.size() ; i++){
     String str = elements.get(i).html();
     list.add(str);
   }
   return list;
 }
 
 private static Log log = LogFactory.getLog(HtmlManage.class);
}

FileDownload.java

    package com.bing.download;

import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileOutputStream;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;

/**
* @说明：
* @author: gaoll
* @CreateTime:2014-11-20
* @ModifyTime:2014-11-20
*/
public class FileDownload {
 
 /**
  * 文件下载
  * @param url 链接地址
  * @param path 要保存的路径及文件名
  * @return
  */
 public static boolean download(String url,String path){
   
   boolean flag = false;
   
   CloseableHttpClient httpclient = HttpClients.createDefault();
   RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(2000)
       .setConnectTimeout(2000).build();

   HttpGet get = new HttpGet(url);
   get.setConfig(requestConfig);
   
   BufferedInputStream in = null;
   BufferedOutputStream out = null;
   try{
     for(int i=0;i<3;i++){
       CloseableHttpResponse result = httpclient.execute(get);
       System.out.println(result.getStatusLine());
       if(result.getStatusLine().getStatusCode() == 200){
         in = new BufferedInputStream(result.getEntity().getContent());
         File file = new File(path);
         out = new BufferedOutputStream(new FileOutputStream(file));
         byte[] buffer = new byte[1024];
         int len = -1;
         while((len = in.read(buffer,0,1024)) > -1){
           out.write(buffer,0,len);
         }
         flag = true;
         break;
       }else if(result.getStatusLine().getStatusCode() == 500){
         continue ;
       }
     }
     
   }catch(Exception e){
     e.printStackTrace();
     flag = false;
   }finally{
     get.releaseConnection();
     try{
       if(in != null){
         in.close();
       }
       if(out != null){
         out.close();
       }
     }catch(Exception e){
       e.printStackTrace();
       flag = false;
     }
   }
   return flag;
 }

 private static Log log = LogFactory.getLog(FileDownload.class);
}

猜你喜欢

转载自blog.csdn.net/qq_36142146/article/details/86234791
今日推荐
基于大语言模型的开源知识库问答系统 MaxKB GitHub Star 数量突破 5,000 个！
美国拟限制 AI 大模型出口中国和俄罗斯
苹果将与 OpenAI 达成协议，将 ChatGPT 应用于 iPhone
openKylin 社区生态委员会第六次会议圆满召开
阿里云正式发布通义千问 2.5
Python 3.13 发布首个 Beta：实验性自由线程模式和 JIT、改进交互式解释器
Stack Overflow 拿我的代码去训练 AI 大模型，还封了我的账号
Pop!_OS 的 COSMIC 桌面完成 App Store 上架工作
报告：Django 仍然是 74% 开发者的首选
《2024 年一季度互联网投融资运行情况》研究报告
15 年前上了“FFmpeg 耻辱柱”，今天他还得谢谢咱——腾讯QQPlayer一雪前耻？
TIOBE 5 月榜单：Fortran “复活”进入 Top 10
周排行
BPM为企业带来的实际利益
好程序员web前端分享css常用属性缩写
Java文件下载（excel）
css样式的动态添加及显示和隐藏等零碎用法
axios全局配置以及拦截器
使用Logstash来实时同步MySQL和log日志数据到ES
C++获取当前时间（年月日、时分秒、毫秒）
Odoo产品分析 (四) -- 工具板块(11) -- 网站即时聊天(1)
Java环境配置正确，但是java、javac、java -version均返回“不是内部或外部命令，也不是可运行的程序或批处理文件”？
01 官网下载各种CentOS教程（超详细版）
每日归档
更多
2024-05-14(0)
2024-05-13(18)
2024-05-12(0)
2024-05-11(38)
2024-05-10(38)
2024-05-09(35)
2024-05-08(42)
2024-05-07(14)
2024-05-06(40)
2024-05-05(0)

代码天地 © 2018 codetd.com

境外服务器   最新电视剧2022