《蜘蛛程序 网络爬虫 源代码》由会员分享，可在线阅读，更多相关《蜘蛛程序 网络爬虫 源代码（7页珍藏版）》请在金锄头文库上搜索。
1、有一天突然看了htmlparser工具包发现果然强大。由于不是很熟悉 所以下面代码或许写的有点烂。首先做准备工作 先写一个实体beanpackage bean; import java.util.Date;/* author Jeson* blog * date:Oct 9, 2009 3:09:19 PM* version :1.0*/public class Artical private String title; private String body; private String link; private String author; private String tags; p
2、rivate Date dCreate; public String getTitle() return title; public void setTitle(String title) this.title = title; public String getBody() return body; public void setBody(String body) this.body = body; public String getLink() return link; public void setLink(String link) this.link = link; public St
3、ring getAuthor() return author; public void setAuthor(String author) this.author = author; public String getTags() return tags; public void setTags(String tags) this.tags = tags; public Date getDCreate() return dCreate; public void setDCreate(Date create) dCreate = create; 2 写一个我们下面要用到的字符串处理类package
4、 util;import java.io.BufferedReader;import java.io.IOException;import java.io.InputStream;import java.io.InputStreamReader;import java.io.UnsupportedEncodingException;import java.util.regex.Matcher;import java.util.regex.Pattern;/* author Jeson* blog * date:Oct 9, 2009 3:09:19 PM * version :1.0*/pub
5、lic class StringUtil /* * 使用正则匹配字符串 * * param regex * 正则表达式 * param txt * 要验证的字符串 * return 匹配则返回真 否则返回假 */ public static boolean useRegex(String regex, String txt) Pattern p = Ppile(regex); Matcher m = p.matcher(txt); return m.matches(); /* * 使用正则匹配字符串 * * param regex * 正则表达式 * * param index * 要取第几个
6、元素 * param txt * 要验证的字符串 * return 返回匹配的字符串 */ public static String getByRegex(String regex, int index, String txt) Pattern p = Ppile(regex); Matcher m = p.matcher(txt); if (m.find() return m.group(index); return null; /* * 使用正则匹配字符串 * * param regex * 正则表达式 * * param index * 要取第几个元素 * param txt * 要验证
7、的字符串 * return 返回匹配的字符串数组 */ public static String getStringsByRegex(String regex, int index, String txt) String res = new Stringindex.length; Pattern p = Ppile(regex); Matcher m = p.matcher(txt); if (m.find() for(int i : index) resi = m.group(i); return res; 3 下面是我们的核心类 他会去抓取cnblogs的页面并保存 package tes
8、t; import java.io.File;import java.io.FileNotFoundException;import java.io.FileOutputStream;import java.io.IOException;import java.io.OutputStream;import org.htmlparser.NodeFilter;import org.htmlparser.Parser;import org.htmlparser.filters.HasAttributeFilter;import org.htmlparser.util.NodeList;import
9、 bean.Artical;import util.StringUtil;/* author Jeson* blog * date:Oct 9, 2009 1:08:10 PM* version :1.0*/public class Parse private static final int MAX_PAGE = 20; private final String ENCODING = UTF-8; /* * param args */ public static void main(String args) try for(int i=1;iMAX_PAGE;i+) new Parse().
10、testAttribute(i); catch (Exception e) / TODO Auto-generated catch block e.printStackTrace(); private void testAttribute(int pa) throws Exception System.out.println(开始解析页面：+pa); Parser p = new Parser(); p.setURL( p.setEncoding(UTF-8); NodeFilter filter = new HasAttributeFilter(class,titlelnk); NodeLi
11、st list = p.extractAllNodesThatMatch(filter); System.out.println(list.size(); int cou = 0; for(int i=0 ; ilist.size();i+) String html = list.elementAt(i).toHtml(true); int index = 0,1,2; String bs = StringUtil.getStringsByRegex(.*), index, html); String title = bs2; String url = bs1; System.out.println(url); String content = getContent(bs1);
