package work;
import org.jsoup.Jsoup;
import org.jsoup.safety.Whitelist;
import org.springframework.dao.DuplicateKeyException;
import org.springframework.jdbc.core.JdbcTemplate;
import cn.edu.hfut.dmic.contentextractor.ContentExtractor;
import cn.edu.hfut.dmic.contentextractor.News;
import cn.edu.hfut.dmic.webcollector.conf.Configuration;
import cn.edu.hfut.dmic.webcollector.model.CrawlDatum;
import cn.edu.hfut.dmic.webcollector.model.CrawlDatums;
import cn.edu.hfut.dmic.webcollector.model.Page;
import cn.edu.hfut.dmic.webcollector.plugin.berkeley.BreadthCrawler;
import cn.edu.hfut.dmic.webcollector.plugin.net.OkHttpRequester;
import db.JDBCHelper;
import okhttp3.Request;
import util.HtmlTools;
/**
* Crawling news from hfut news
* use 2.72 lib
* @author hu
*/
public class ChujiingNewstpl extends BreadthCrawler {
//种子 url
public String seedUrl="http://news.cnhubei.com/";
//需要采集的内容页url
public String contentRegUrl="http://news.cnhubei.com/.*/p/.*?.html\\.*";
//线程数量
public int threads_num=10;
//每次迭代爬取的网页数量上限
public int topn_num=10;
//爬取文章深度
public static int levelnum=10;
//停止后能否继续上次采集
public static boolean resumable=true;
public int executeTime=20000; //ms
public static int MaxExecuteCount=2;
public int connectTimeout=50;
public int readTimeout=60;
private String contentTable="news_content";
@Override
public void visit(Page page, CrawlDatums next) {
// String url = page.url();
if (page.matchUrl(contentRegUrl)) {
//
/*extract title and content of news by css selector*/
// String title = page.select("div[id=Article]>h3").first().text();
// String content = page.selectText("div#artibody");
News n = null;
try {
n=ContentExtractor.getNewsByHtml(page.html());
String title=n.getTitle();
String content=n.getContent();
content = Jsoup.clean(content, HtmlTools.getWhitelist());
content=HtmlTools.stripNewLine(content);
title=Jsoup.clean(title,Whitelist.none());
title=title.trim();
System.out.println(" get content :"+title );
if(!title.isEmpty() && !content.isEmpty()) {
ChujiingNewstpl.dbHandler.update("insert into "+contentTable+"(title,content) value(?,?)",title,content);
}
} catch(DuplicateKeyException e) {
System.out.println(" duplicate item ");
}catch (Exception e) {
// TODO Auto-generated catch block
System.out.println(e.getMessage());
}
}
}
private static JdbcTemplate dbHandler;
// 自定义的请求插件
public class MyRequester extends OkHttpRequester {
String userAgent = "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)";
// String cookie = "name=abcdef";
// 每次发送请求前都会执行这个方法来构建请求
@Override
public Request.Builder createRequestBuilder(CrawlDatum crawlDatum) {
// 这里使用的是OkHttp中的Request.Builder
// 可以参考OkHttp的文档来修改请求头
// System.out.println("request with cookie: " + cookie);
return super.createRequestBuilder(crawlDatum).header("User-Agent", userAgent);
// .header("Cookie", cookie);
}
}
public ChujiingNewstpl(String crawlPath, boolean autoParse) {
super(crawlPath, autoParse);
// 设置请求插件
//setRequester(new MyRequester());
/*start page*/
this.addSeed(seedUrl);
this.addRegex(contentRegUrl);
this.addRegex("-.*\\.(jpg|png|gif|css|js|font).*");
setThreads(threads_num);
Configuration cnf=getConf();
cnf.setTopN(topn_num);
// cnf.setExecuteInterval(executeTime);
// cnf.setConnectTimeout(connectTimeout);
// cnf.setReadTimeout(readTimeout);
}
public static void main(String[] args) throws Exception {
dbHandler=JDBCHelper.db();
ChujiingNewstpl crawler = new ChujiingNewstpl("spiderdata"+java.io.File.separator+ChujiingNewstpl.class.getName(), true);
crawler.setResumable(resumable);
crawler.start(levelnum);
//失败最大尝试次数
crawler.setMaxExecuteCount(MaxExecuteCount);
}
}
源码地址 https://down.51cto.com/data/2461609
免责声明:本站发布的内容(图片、视频和文字)以原创、转载和分享为主,文章观点不代表本网站立场,如果涉及侵权请联系站长邮箱:is@yisu.com进行举报,并提供相关证据,一经查实,将立刻删除涉嫌侵权内容。