JAVA编程131——httpclient+jsoup爬一篇小说 - 好文

<>一、maven坐标
<dependencies>  <
dependency> <groupId>org.apache.httpcomponents</groupId> <artifactId>httpclient
</artifactId> <version>4.5.2</version> </dependency>  <dependency> <
groupId>org.slf4j</groupId> <artifactId>slf4j-log4j12</artifactId> <version>
1.7.25</version>  </dependency>  <dependency> <groupId>
org.jsoup</groupId> <artifactId>jsoup</artifactId> <version>1.10.2</version> </
dependency>  <dependency>
<groupId>junit</groupId> <artifactId>junit</artifactId> <version>4.12</version>
</dependency> </dependencies>
<>二、demo
package com.mollen; import org.apache.http.HttpEntity; import org.apache.http.
client.methods.CloseableHttpResponse; import org.apache.http.client.methods.
HttpGet; import org.apache.http.impl.client.CloseableHttpClient; import org.
apache.http.impl.client.HttpClients; import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.
Element; import org.jsoup.select.Elements; import org.junit.Test; import java.
nio.charset.Charset; /** * @ClassName: demo * @Auther: Mollen * @CreateTime:
2019-01-15 19:27:20 * @Description: */ public class demo { @Test public void
getContent() throws Exception { String url =
"https://read.qidian.com/chapter/_h17RCSkeXScikCo3ZPkrg2/_LrKPdZirvtMs5iq0oQwLQ2"
; while(url!=null){ //1.发送一个请求 HttpGet httpGet = new HttpGet(url);
CloseableHttpClient httpClient= HttpClients.createDefault(); //2.获得响应
CloseableHttpResponse response= httpClient.execute(httpGet); HttpEntity entity =
response.getEntity(); String html = EntityUtils.toString(entity, Charset.
forName("utf-8")); //3.解析章节内容 Document doc = Jsoup.parse(html); //3.1 获取章节名称
Element name= doc.select(".j_chapterName").get(0); System.out.println(
"------------------------------------------------------------------------");
System.out.println(name.text()); //3.2 获取章节内容 Elements contents = doc.select(
"div[class=read-content j_readContent] p"); for (Element content : contents) {
System.out.println(content.text()); } //3.3 获取下一个章节的url Elements nextUrls = doc.
select("#j_chapterNext"); Element nextUrl = nextUrls.get(0); url = "https:"+
nextUrl.attr("href"); } } }

热门工具换一换