search
HomeJavajavaTutorialJava multi-threading captures ringtone data from the official website of Ringtone Duoduo

一直想练习下java多线程抓取数据。

有天被我发现,铃声多多的官网(http://www.shoujiduoduo.com/main/)有大量的数据。

通过观察他们前端获取铃声数据的ajax

Java multi-threading captures ringtone data from the official website of Ringtone Duoduo

http://www.shoujiduoduo.com/ringweb/ringweb.php?type=getlist&listid={类别ID}&page={分页页码}

很容易就能发现通过改变 listId和page就能从服务器获取铃声的json数据, 通过解析json数据,

可以看到都带有{"hasmore":1,"curpage":1}这样子的指示,通过判断hasmore的值,决定是否进行下一页的抓取。

但是通过上面这个链接返回的json中不带有铃声的下载地址

很快就可以发现,点击页面的“下载”会看到

通过下面的请求,就可以获取铃声的下载地址了

http://www.shoujiduoduo.com/ringweb/ringweb.php?type=geturl&act=down&rid={铃声ID}

Java multi-threading captures ringtone data from the official website of Ringtone Duoduo

所以,他们的数据是很容易被偷的。于是我就开始...

源码已经发在github上。如果感兴趣的童鞋可以查看

github:https://github.com/yongbo000/DuoduoAudioRobot

上代码:

<pre class="brush:java;">package me.yongbo.DuoduoRingRobot;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.URL;
import java.net.URLConnection;
import java.util.Iterator;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import com.google.gson.Gson;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonParser;
/* * @author yongbo_ * @created 2013/4/16 * * */
public class DuoduoRingRobotClient implements Runnable {
public static String GET_RINGINFO_URL = "http://www.shoujiduoduo.com/ringweb/ringweb.php?type=getlist&listid=%1$d&page=%2$d";
public static String GET_DOWN_URL = "http://www.shoujiduoduo.com/ringweb/ringweb.php?type=geturl&act=down&rid=%1$d";
public static String ERROR_MSG = "listId为 %1$d 的Robot发生错误,已自动停止。当前page为 %2$d";public static String STATUS_MSG = "开始抓取数据,当前listId: %1$d,当前page: %2$d";
public static String FILE_DIR = "E:/RingData/";public static String FILE_NAME = "listId=%1$d.txt";private boolean errorFlag = false;private int listId;private int page;
private int endPage = -1;private int hasMore = 1;
private DbHelper dbHelper;
/** * 构造函数 * @param listId 菜单ID * @param page 开始页码 * @param endPage 结束页码 * */
public DuoduoRingRobotClient(int listId, int beginPage, int endPage)
 {this.listId = listId;this.page = beginPage;this.endPage = endPage;this.dbHelper = new DbHelper();}
/** * 构造函数 * @param listId 菜单ID * @param page 开始页码 * */
public DuoduoRingRobotClient(int listId, int page) {this(listId, page, -1);}
/** * 获取铃声 * */public void getRings() {String url = String.format(GET_RINGINFO_URL, listId, page);String responseStr = httpGet(url);hasMore = getHasmore(responseStr);
page = getNextPage(responseStr);
ringParse(responseStr.replaceAll("\\{\"hasmore\":[0-9]*,\"curpage\":[0-9]*\\},", "").replaceAll(",]", "]"));}/** * 发起http请求 * @param webUrl 请求连接地址 * */public String httpGet(String webUrl){URL url;URLConnection conn;StringBuilder sb = new StringBuilder();String resultStr = "";try {url = new URL(webUrl);conn = url.openConnection();conn.connect();InputStream is = conn.getInputStream();InputStreamReader isr = new InputStreamReader(is);BufferedReader bufReader = new BufferedReader(isr);String lineText;while ((lineText = bufReader.readLine()) != null) {sb.append(lineText);}resultStr = sb.toString();} catch (Exception e) {errorFlag = true;//将错误写入txtwriteToFile(String.format(ERROR_MSG, listId, page));}return resultStr;}/** * 将json字符串转化成Ring对象,并存入txt中 * @param json Json字符串 * */public void ringParse(String json) {Ring ring = null;JsonElement element = new JsonParser().parse(json);JsonArray array = element.getAsJsonArray();// 遍历数组Iterator<JsonElement> it = array.iterator();
Gson gson = new Gson();while (it.hasNext() && !errorFlag) {JsonElement e = it.next();// JsonElement转换为JavaBean对象ring = gson.fromJson(e, Ring.class);ring.setDownUrl(getRingDownUrl(ring.getId()));if(isAvailableRing(ring)) {System.out.println(ring.toString());
//可选择写入数据库还是写入文本//writeToFile(ring.toString());writeToDatabase(ring);}}}
/** * 写入txt * @param data 字符串 * */public void writeToFile(String data)
 {String path = FILE_DIR + String.format(FILE_NAME, listId);File dir = new File(FILE_DIR);File file = new File(path);FileWriter fw = null;if(!dir.exists()){dir.mkdirs();
}try {if(!file.exists()){file.createNewFile();}fw = new FileWriter(file, true);
fw.write(data);fw.write("\r\n");fw.flush();} catch (IOException e) {
// TODO Auto-generated catch blocke.printStackTrace();
}finally {try {if(fw != null){fw.close();}} catch (IOException e) {
// TODO Auto-generated catch blocke.printStackTrace();}}}/** * 写入数据库 * @param ring 一个Ring的实例 * */
public void writeToDatabase(Ring ring) {dbHelper.execute("addRing", ring);}
@Overridepublic void run() {while(hasMore == 1 && !errorFlag){if(endPage != -1){if(page > endPage) { break; }}System.out.println(String.format(STATUS_MSG, listId, page));
getRings();System.out.println(String.format("该页数据写入完成"));}System.out.println("ending...");}
private int getHasmore(String resultStr){Pattern p = Pattern.compile("\"hasmore\":([0-9]*),\"curpage\":([0-9]*)"); 
 Matcher match = p.matcher(resultStr);  
 if (match.find()) {  return Integer.parseInt(match.group(1));
  }  return 0;
}
private int getNextPage(String resultStr){Pattern p = Pattern.compile("\"hasmore\":([0-9]*),\"curpage\":([0-9]*)");Matcher match = p.matcher(resultStr);if (match.find()) {return Integer.parseInt(match.group(2));}return 0;}
/** * 判断当前Ring是否满足条件。当Ring的name大于50个字符或是duration为小数则不符合条件,将被剔除。 * @param ring 当前Ring对象实例 * */private boolean isAvailableRing(Ring ring){Pattern p = Pattern.compile("^[1-9][0-9]*$");
Matcher match = p.matcher(ring.getDuration());
if(!match.find()){return false;}if(ring.getName().length() > 50 || ring.getArtist().length() > 50 || ring.getDownUrl().length() == 0){return false;}return true;}
/** * 获取铃声的下载地址 * @param rid 铃声的id * */
public String getRingDownUrl(String rid){String url = String.format(GET_DOWN_URL, rid);
String responseStr = httpGet(url);return responseStr;}}

更多Java multi-threading captures ringtone data from the official website of Ringtone Duoduo相关文章请关注PHP中文网!

Statement
The content of this article is voluntarily contributed by netizens, and the copyright belongs to the original author. This site does not assume corresponding legal responsibility. If you find any content suspected of plagiarism or infringement, please contact admin@php.cn
How does IntelliJ IDEA identify the port number of a Spring Boot project without outputting a log?How does IntelliJ IDEA identify the port number of a Spring Boot project without outputting a log?Apr 19, 2025 pm 11:45 PM

Start Spring using IntelliJIDEAUltimate version...

How to elegantly obtain entity class variable names to build database query conditions?How to elegantly obtain entity class variable names to build database query conditions?Apr 19, 2025 pm 11:42 PM

When using MyBatis-Plus or other ORM frameworks for database operations, it is often necessary to construct query conditions based on the attribute name of the entity class. If you manually every time...

How to use the Redis cache solution to efficiently realize the requirements of product ranking list?How to use the Redis cache solution to efficiently realize the requirements of product ranking list?Apr 19, 2025 pm 11:36 PM

How does the Redis caching solution realize the requirements of product ranking list? During the development process, we often need to deal with the requirements of rankings, such as displaying a...

How to safely convert Java objects to arrays?How to safely convert Java objects to arrays?Apr 19, 2025 pm 11:33 PM

Conversion of Java Objects and Arrays: In-depth discussion of the risks and correct methods of cast type conversion Many Java beginners will encounter the conversion of an object into an array...

How do I convert names to numbers to implement sorting and maintain consistency in groups?How do I convert names to numbers to implement sorting and maintain consistency in groups?Apr 19, 2025 pm 11:30 PM

Solutions to convert names to numbers to implement sorting In many application scenarios, users may need to sort in groups, especially in one...

E-commerce platform SKU and SPU database design: How to take into account both user-defined attributes and attributeless products?E-commerce platform SKU and SPU database design: How to take into account both user-defined attributes and attributeless products?Apr 19, 2025 pm 11:27 PM

Detailed explanation of the design of SKU and SPU tables on e-commerce platforms This article will discuss the database design issues of SKU and SPU in e-commerce platforms, especially how to deal with user-defined sales...

How to set the default run configuration list of SpringBoot projects in Idea for team members to share?How to set the default run configuration list of SpringBoot projects in Idea for team members to share?Apr 19, 2025 pm 11:24 PM

How to set the SpringBoot project default run configuration list in Idea using IntelliJ...

See all articles

Hot AI Tools

Undresser.AI Undress

Undresser.AI Undress

AI-powered app for creating realistic nude photos

AI Clothes Remover

AI Clothes Remover

Online AI tool for removing clothes from photos.

Undress AI Tool

Undress AI Tool

Undress images for free

Clothoff.io

Clothoff.io

AI clothes remover

Video Face Swap

Video Face Swap

Swap faces in any video effortlessly with our completely free AI face swap tool!

Hot Tools

MantisBT

MantisBT

Mantis is an easy-to-deploy web-based defect tracking tool designed to aid in product defect tracking. It requires PHP, MySQL and a web server. Check out our demo and hosting services.

Dreamweaver Mac version

Dreamweaver Mac version

Visual web development tools

SublimeText3 Mac version

SublimeText3 Mac version

God-level code editing software (SublimeText3)

PhpStorm Mac version

PhpStorm Mac version

The latest (2018.2.1) professional PHP integrated development tool

WebStorm Mac version

WebStorm Mac version

Useful JavaScript development tools