传统爬虫从一个或若干初始网页的URL开始,获得初始网页上的URL,在抓取网页的过程中,不断从当前页面上抽取新的URL放入队列,直到满足系统的一定停止条件。对于垂直搜索来说,聚焦爬虫,即有针对性地爬取特定主题网页的爬虫,更为适合。
以下是一个使用java实现的简单爬虫核心代码:
public void crawl() throws Throwable {
while (continueCrawling()) {
CrawlerUrl url = getNextUrl()//获取待爬取队列中的下一个URL
if (url != null) {
printCrawlInfo()
String content = getContent(url)//获取URL的文本信息
//聚焦爬虫只爬取与主题内容相关的网页,这里采用正则匹配简单处理
if (isContentRelevant(content, this.regexpSearchPattern)) {
saveContent(url, content)//保存网页至本地
//获取网页内容中的链接,并放入待爬取队列中
Collection urlStrings = extractUrls(content, url)
addUrlsToUrlQueue(url, urlStrings)
} else {
System.out.println(url + " is not relevant ignoring ...")
}
//延时防止被对方屏蔽
Thread.sleep(this.delayBetweenUrls)
}
}
closeOutputStream()
}
private CrawlerUrl getNextUrl() throws Throwable {
CrawlerUrl nextUrl = null
while ((nextUrl == null) &&(!urlQueue.isEmpty())) {
CrawlerUrl crawlerUrl = this.urlQueue.remove()
//doWeHavePermissionToVisit:是否有权限访问该URL,友好的爬虫会根据网站提供的"Robot.txt"中配置的规则进行爬取
//isUrlAlreadyVisited:URL是否访问过,大型的搜索引擎往往采用BloomFilter进行排重,这里简单使用HashMap
//isDepthAcceptable:是否达到指定的深度上限。爬虫一般采取广度优先的方式。一些网站会构建爬虫陷阱(自动生成一些无效链接使爬虫陷入死循环),采用深度限制加以避免
if (doWeHavePermissionToVisit(crawlerUrl)
&&(!isUrlAlreadyVisited(crawlerUrl))
&&isDepthAcceptable(crawlerUrl)) {
nextUrl = crawlerUrl
// System.out.println("Next url to be visited is " + nextUrl)
}
}
return nextUrl
}
private String getContent(CrawlerUrl url) throws Throwable {
//HttpClient4.1的调用与之前的方式不同
HttpClient client = new DefaultHttpClient()
HttpGet httpGet = new HttpGet(url.getUrlString())
StringBuffer strBuf = new StringBuffer()
HttpResponse response = client.execute(httpGet)
if (HttpStatus.SC_OK == response.getStatusLine().getStatusCode()) {
HttpEntity entity = response.getEntity()
if (entity != null) {
BufferedReader reader = new BufferedReader(
new InputStreamReader(entity.getContent(), "UTF-8"))
String line = null
if (entity.getContentLength() >0) {
strBuf = new StringBuffer((int) entity.getContentLength())
while ((line = reader.readLine()) != null) {
strBuf.append(line)
}
}
}
if (entity != null) {
nsumeContent()
}
}
//将url标记为已访问
markUrlAsVisited(url)
return strBuf.toString()
}
public static boolean isContentRelevant(String content,
Pattern regexpPattern) {
boolean retValue = false
if (content != null) {
//是否符合正则表达式的条件
Matcher m = regexpPattern.matcher(content.toLowerCase())
retValue = m.find()
}
return retValue
}
public List extractUrls(String text, CrawlerUrl crawlerUrl) {
Map urlMap = new HashMap()
extractHttpUrls(urlMap, text)
extractRelativeUrls(urlMap, text, crawlerUrl)
return new ArrayList(urlMap.keySet())
}
private void extractHttpUrls(Map urlMap, String text) {
Matcher m = (text)
while (m.find()) {
String url = m.group()
String[] terms = url.split("a href=\"")
for (String term : terms) {
// System.out.println("Term = " + term)
if (term.startsWith("http")) {
int index = term.indexOf("\"")
if (index >0) {
term = term.substring(0, index)
}
urlMap.put(term, term)
System.out.println("Hyperlink: " + term)
}
}
}
}
private void extractRelativeUrls(Map urlMap, String text,
CrawlerUrl crawlerUrl) {
Matcher m = relativeRegexp.matcher(text)
URL textURL = crawlerUrl.getURL()
String host = textURL.getHost()
while (m.find()) {
String url = m.group()
String[] terms = url.split("a href=\"")
for (String term : terms) {
if (term.startsWith("/")) {
int index = term.indexOf("\"")
if (index >0) {
term = term.substring(0, index)
}
String s = //" + host + term
urlMap.put(s, s)
System.out.println("Relative url: " + s)
}
}
}
}
public static void main(String[] args) {
try {
String url = ""
Queue urlQueue = new LinkedList()
String regexp = "java"
urlQueue.add(new CrawlerUrl(url, 0))
NaiveCrawler crawler = new NaiveCrawler(urlQueue, 100, 5, 1000L,
regexp)
// boolean allowCrawl = crawler.areWeAllowedToVisit(url)
// System.out.println("Allowed to crawl: " + url + " " +
// allowCrawl)
crawler.crawl()
} catch (Throwable t) {
System.out.println(t.toString())
t.printStackTrace()
}
}
1,网络机器人Java编程指南,浅显易懂,有点过时,但适合新手2,自己动手写网络爬虫,有点基础还可以看看,写的有点乱,很多内容交代不清楚,并且大篇幅代码抄袭。。。
3,搜索引擎 ——原理、技术与系统,北大天网为案例,很好很强大,有点学术味道
4,Web数据挖掘 Bing Liu,刘兵的书,强烈推荐
5,搜索引擎:信息检索实践,很好的书,强烈推荐
还有一些论文,自己去找吧
案例的话,可以研究下Nutch爬虫部分代码,写的很清晰
有了以上这些,应该算是入门了~
import java.awt.*import java.awt.event.*
import java.io.*
import java.net.*
import java.util.*
import java.util.regex.*
import javax.swing.*
import javax.swing.table.*//一个Web的爬行者(注:爬行在这里的意思与抓取,捕获相同)
public class SearchCrawler extends JFrame{
//最大URL保存值
private static final String[] MAX_URLS={"50","100","500","1000"}
//缓存robot禁止爬行列表
private HashMap disallowListCache=new HashMap()
//搜索GUI控件
private JTextField startTextField
private JComboBox maxComboBox
private JCheckBox limitCheckBox
private JTextField logTextField
private JTextField searchTextField
private JCheckBox caseCheckBox
private JButton searchButton
//搜索状态GUI控件
private JLabel crawlingLabel2
private JLabel crawledLabel2
private JLabel toCrawlLabel2
private JProgressBar progressBar
private JLabel matchesLabel2
//搜索匹配项表格列表
private JTable table
//标记爬行机器是否正在爬行
private boolean crawling
//写日志匹配文件的引用
private PrintWriter logFileWriter
//网络爬行者的构造函数
public SearchCrawler(){
//设置应用程序标题栏
setTitle("搜索爬行者")
//设置窗体大小
setSize(600,600)
//处理窗体关闭事件
addWindowListener(new WindowAdapter(){
public void windowClosing(WindowEvent e){
actionExit()
}
})
//设置文件菜单
JMenuBar menuBar=new JMenuBar()
JMenu fileMenu=new JMenu("文件")
fileMenu.setMnemonic(KeyEvent.VK_F)
JMenuItem fileExitMenuItem=new JMenuItem("退出",KeyEvent.VK_X)
fileExitMenuItem.addActionListener(new ActionListener(){
public void actionPerformed(ActionEvent e){
actionExit()
}
})
fileMenu.add(fileExitMenuItem)
menuBar.add(fileMenu)
setJMenuBar(menuBar)