网络爬虫java代码网络爬虫编程

小程序：扫一扫查出行
【扫一扫了解最新限行尾号】
复制小程序

求用JAVA编写一个网络爬虫的程序

先根据一个链接，抓取该页面；解析该页面，搜取出该页面中有用的链接，根据链接地址循环抓取就OK了；--简单爬行器原理抓取完文件后还需要一个功能好点的文档解析器来解析出文件中的内容；--文件解析器再根据关键字（分词器）处理这些页面，建立自己的搜索引擎；--分词器不好解决我也正在搞这方面呵呵

Java源码实现网络爬虫?

//Java爬虫demo

import java.io.File;

import java.net.URL;

import java.net.URLConnection;

import java.nio.file.Files;

import java.nio.file.Paths;

import java.util.Scanner;

import java.util.UUID;

import java.util.regex.Matcher;

import java.util.regex.Pattern;

public class DownMM {

public static void main(String[] args) throws Exception {

//out为输出的路径,注意要以\\结尾

String out = "D:\\JSP\\pic\\java\\";

try{

File f = new File(out);

if(! f.exists()) {

f.mkdirs();

}

}catch(Exception e){

System.out.println("no");

}

String url = "-";

Pattern reg = Pattern.compile("img src=\"(.*?)\"");

for(int j=0, i=1; i=10; i++){

URL uu = new URL(url+i);

URLConnection conn = uu.openConnection();

conn.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; rv:11.0) like Gecko");

Scanner sc = new Scanner(conn.getInputStream());

Matcher m = reg.matcher(sc.useDelimiter("\\A").next());

while(m.find()){

Files.copy(new URL(m.group(1)).openStream(), Paths.get(out + UUID.randomUUID() + ".jpg"));

System.out.println("已下载:"+j++);

}

用java编写网络爬虫，用来爬网络音乐资源，再返回java页面显示该怎么实现

下面是源代码,希望可以帮到你~~

package com.ly.mainprocess;

import java.io.BufferedReader;

import java.io.InputStreamReader;

import java.util.ArrayList;

import java.util.List;

import org.apache.http.Consts;

import org.apache.http.Header;

import org.apache.http.HttpEntity;

import org.apache.http.HttpResponse;

import org.apache.http.NameValuePair;

import org.apache.http.StatusLine;

import org.apache.http.client.entity.UrlEncodedFormEntity;

import org.apache.http.client.methods.HttpGet;

import org.apache.http.client.methods.HttpPost;

import org.apache.http.cookie.Cookie;

import org.apache.http.impl.client.DefaultHttpClient;

import org.apache.http.message.BasicNameValuePair;

import org.apache.http.util.EntityUtils;

public class Test1 {

public static void main(String[] args){

Test1 test1 = new Test1();

System.out.println(test1.process("******","******"));

}

@SuppressWarnings("deprecation")

public boolean process(String username,String password) {

boolean ret=false;

DefaultHttpClient httpclient = new DefaultHttpClient();

try {

HttpGet httpget;

HttpResponse response;

HttpEntity entity;

ListCookie cookies;

//组建登录的post包

HttpPost httppost = new HttpPost(""); // 用户登录

ListNameValuePair nvps = new ArrayListNameValuePair();

nvps.add(new BasicNameValuePair("nickname", username));

nvps.add(new BasicNameValuePair("password", password));

nvps.add(new BasicNameValuePair("origURL", ""));

nvps.add(new BasicNameValuePair("loginregFrom", "index"));

nvps.add(new BasicNameValuePair("ss", "10101"));

httppost.setEntity(new UrlEncodedFormEntity(nvps, Consts.UTF_8));

httppost.addHeader("Referer", "");

httppost.addHeader("Connection", "keep-alive");

httppost.addHeader("Content-Type", "application/x-www-form-urlencoded");

httppost.addHeader("Accept-Language", "zh-CN,zh;q=0.8");

httppost.addHeader("Origin", "");

httppost.addHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36");

response = httpclient.execute(httppost);

entity = response.getEntity();

// System.out.println("Login form get: " + response.getStatusLine());

EntityUtils.consume(entity);

// System.out.println("Post logon cookies:");

cookies = httpclient.getCookieStore().getCookies();

if (cookies.isEmpty()) {

// System.out.println("None");

} else {

for (int i = 0; i cookies.size(); i++) {

// System.out.println("- " + cookies.get(i).toString());

}

//进行页面跳转

String url = ""; // 页面跳转

Header locationHeader = response.getFirstHeader("Location");

// System.out.println(locationHeader.getValue());

if (locationHeader != null) {

url = locationHeader.getValue(); // 得到跳转href

HttpGet httpget1 = new HttpGet(url);

response = httpclient.execute(httpget1);

// 登陆成功。。。hoho

}

entity = response.getEntity();

// System.out.println(response.getStatusLine());

if (entity != null) {

// System.out.println("Response content length: " + entity.getContentLength());

}

// 显示结果

BufferedReader reader = new BufferedReader(new InputStreamReader(entity.getContent(), "UTF-8"));

String line = null;

while ((line = reader.readLine()) != null) {

// System.out.println(line);

}

//自动打卡

// 访问网站的子网页。

HttpPost httppost1 = new HttpPost(""); // 设置个人信息页面

httppost1.addHeader("Content-Type", "text/plain;charset=UTF-8");

httppost1.addHeader("Accept", "text/plain, */*");

httppost1.addHeader("X-Requested-With", "XMLHttpRequest");

httppost1.addHeader("Referer", "");

response = httpclient.execute(httppost1);

entity = response.getEntity();

// System.out.println(response.getStatusLine());

if(response.getStatusLine().toString().indexOf("HTTP/1.1 200 OK")=0){

ret = true;

}

if (entity != null) {

// System.out.println("Response content length: " + entity.getContentLength());

}

// 显示结果

reader = new BufferedReader(new InputStreamReader(entity.getContent(), "UTF-8"));

line = null;

while ((line = reader.readLine()) != null) {

System.out.println(line);

}

} catch (Exception e) {

} finally {

httpclient.getConnectionManager().shutdown();

}

return ret;

}

Java网络爬虫怎么实现？

网络爬虫是一个自动提取网页的程序，它为搜索引擎从万维网上下载网页，是搜索引擎的重要组成。\x0d\x0a传统爬虫从一个或若干初始网页的URL开始，获得初始网页上的URL，在抓取网页的过程中，不断从当前页面上抽取新的URL放入队列，直到满足系统的一定停止条件。对于垂直搜索来说，聚焦爬虫，即有针对性地爬取特定主题网页的爬虫，更为适合。\x0d\x0a\x0d\x0a以下是一个使用java实现的简单爬虫核心代码：\x0d\x0apublic void crawl() throws Throwable { \x0d\x0a while (continueCrawling()) { \x0d\x0a CrawlerUrl url = getNextUrl(); //获取待爬取队列中的下一个URL \x0d\x0a if (url != null) { \x0d\x0a printCrawlInfo(); \x0d\x0a String content = getContent(url); //获取URL的文本信息 \x0d\x0a \x0d\x0a //聚焦爬虫只爬取与主题内容相关的网页，这里采用正则匹配简单处理 \x0d\x0a if (isContentRelevant(content, this.regexpSearchPattern)) { \x0d\x0a saveContent(url, content); //保存网页至本地 \x0d\x0a \x0d\x0a //获取网页内容中的链接，并放入待爬取队列中 \x0d\x0a Collection urlStrings = extractUrls(content, url); \x0d\x0a addUrlsToUrlQueue(url, urlStrings); \x0d\x0a } else { \x0d\x0a System.out.println(url + " is not relevant ignoring ..."); \x0d\x0a } \x0d\x0a \x0d\x0a //延时防止被对方屏蔽 \x0d\x0a Thread.sleep(this.delayBetweenUrls); \x0d\x0a } \x0d\x0a } \x0d\x0a closeOutputStream(); \x0d\x0a}\x0d\x0aprivate CrawlerUrl getNextUrl() throws Throwable { \x0d\x0a CrawlerUrl nextUrl = null; \x0d\x0a while ((nextUrl == null) (!urlQueue.isEmpty())) { \x0d\x0a CrawlerUrl crawlerUrl = this.urlQueue.remove(); \x0d\x0a //doWeHavePermissionToVisit：是否有权限访问该URL，友好的爬虫会根据网站提供的"Robot.txt"中配置的规则进行爬取 \x0d\x0a //isUrlAlreadyVisited：URL是否访问过，大型的搜索引擎往往采用BloomFilter进行排重，这里简单使用HashMap \x0d\x0a //isDepthAcceptable：是否达到指定的深度上限。爬虫一般采取广度优先的方式。一些网站会构建爬虫陷阱（自动生成一些无效链接使爬虫陷入死循环），采用深度限制加以避免 \x0d\x0a if (doWeHavePermissionToVisit(crawlerUrl) \x0d\x0a (!isUrlAlreadyVisited(crawlerUrl)) \x0d\x0a isDepthAcceptable(crawlerUrl)) { \x0d\x0a nextUrl = crawlerUrl; \x0d\x0a // System.out.println("Next url to be visited is " + nextUrl); \x0d\x0a } \x0d\x0a } \x0d\x0a return nextUrl; \x0d\x0a}\x0d\x0aprivate String getContent(CrawlerUrl url) throws Throwable { \x0d\x0a //HttpClient4.1的调用与之前的方式不同 \x0d\x0a HttpClient client = new DefaultHttpClient(); \x0d\x0a HttpGet httpGet = new HttpGet(url.getUrlString()); \x0d\x0a StringBuffer strBuf = new StringBuffer(); \x0d\x0a HttpResponse response = client.execute(httpGet); \x0d\x0a if (HttpStatus.SC_OK == response.getStatusLine().getStatusCode()) { \x0d\x0a HttpEntity entity = response.getEntity(); \x0d\x0a if (entity != null) { \x0d\x0a BufferedReader reader = new BufferedReader( \x0d\x0a new InputStreamReader(entity.getContent(), "UTF-8")); \x0d\x0a String line = null; \x0d\x0a if (entity.getContentLength() 0) { \x0d\x0a strBuf = new StringBuffer((int) entity.getContentLength()); \x0d\x0a while ((line = reader.readLine()) != null) { \x0d\x0a strBuf.append(line); \x0d\x0a } \x0d\x0a } \x0d\x0a } \x0d\x0a if (entity != null) { \x0d\x0a nsumeContent(); \x0d\x0a } \x0d\x0a } \x0d\x0a //将url标记为已访问 \x0d\x0a markUrlAsVisited(url); \x0d\x0a return strBuf.toString(); \x0d\x0a}\x0d\x0apublic static boolean isContentRelevant(String content, \x0d\x0aPattern regexpPattern) { \x0d\x0a boolean retValue = false; \x0d\x0a if (content != null) { \x0d\x0a //是否符合正则表达式的条件 \x0d\x0a Matcher m = regexpPattern.matcher(content.toLowerCase()); \x0d\x0a retValue = m.find(); \x0d\x0a } \x0d\x0a return retValue; \x0d\x0a}\x0d\x0apublic List extractUrls(String text, CrawlerUrl crawlerUrl) { \x0d\x0a Map urlMap = new HashMap(); \x0d\x0a extractHttpUrls(urlMap, text); \x0d\x0a extractRelativeUrls(urlMap, text, crawlerUrl); \x0d\x0a return new ArrayList(urlMap.keySet()); \x0d\x0a} \x0d\x0aprivate void extractHttpUrls(Map urlMap, String text) { \x0d\x0a Matcher m = (text); \x0d\x0a while (m.find()) { \x0d\x0a String url = m.group(); \x0d\x0a String[] terms = url.split("a href=\""); \x0d\x0a for (String term : terms) { \x0d\x0a // System.out.println("Term = " + term); \x0d\x0a if (term.startsWith("http")) { \x0d\x0a int index = term.indexOf("\""); \x0d\x0a if (index 0) { \x0d\x0a term = term.substring(0, index); \x0d\x0a } \x0d\x0a urlMap.put(term, term); \x0d\x0a System.out.println("Hyperlink: " + term); \x0d\x0a } \x0d\x0a } \x0d\x0a } \x0d\x0a} \x0d\x0aprivate void extractRelativeUrls(Map urlMap, String text, \x0d\x0a CrawlerUrl crawlerUrl) { \x0d\x0a Matcher m = relativeRegexp.matcher(text); \x0d\x0a URL textURL = crawlerUrl.getURL(); \x0d\x0a String host = textURL.getHost(); \x0d\x0a while (m.find()) { \x0d\x0a String url = m.group(); \x0d\x0a String[] terms = url.split("a href=\""); \x0d\x0a for (String term : terms) { \x0d\x0a if (term.startsWith("/")) { \x0d\x0a int index = term.indexOf("\""); \x0d\x0a if (index 0) { \x0d\x0a term = term.substring(0, index); \x0d\x0a } \x0d\x0a String s = //" + host + term; \x0d\x0a urlMap.put(s, s); \x0d\x0a System.out.println("Relative url: " + s); \x0d\x0a } \x0d\x0a } \x0d\x0a } \x0d\x0a \x0d\x0a}\x0d\x0apublic static void main(String[] args) { \x0d\x0a try { \x0d\x0a String url = ""; \x0d\x0a Queue urlQueue = new LinkedList(); \x0d\x0a String regexp = "java"; \x0d\x0a urlQueue.add(new CrawlerUrl(url, 0)); \x0d\x0a NaiveCrawler crawler = new NaiveCrawler(urlQueue, 100, 5, 1000L, \x0d\x0a regexp); \x0d\x0a // boolean allowCrawl = crawler.areWeAllowedToVisit(url); \x0d\x0a // System.out.println("Allowed to crawl: " + url + " " + \x0d\x0a // allowCrawl); \x0d\x0a crawler.crawl(); \x0d\x0a } catch (Throwable t) { \x0d\x0a System.out.println(t.toString()); \x0d\x0a t.printStackTrace(); \x0d\x0a } \x0d\x0a}