PHP 网页爬虫
只能爬一个页面
<?php function get_urls($url){ $url_array=array(); $the_first_content=file_get_contents($url); $the_second_content=file_get_contents($url); $pattern1 = "/http:\/\/[a-zA-Z0-9\.\?\/\-\=\&\:\+\-\_\\'\"]+/"; $pattern2="/http:\/\/[a-zA-Z0-9\.]+/"; preg_match_all($pattern2, $the_second_content, $matches2); preg_match_all($pattern1, $the_first_content, $matches1); $new_array1=array_unique($matches1[0]); $new_array2=array_unique($matches2[0]); $final_array=array_merge($new_array1,$new_array2); $final_array=array_unique($final_array); for($i=0;$i<count($final_array);$i++) { echo $final_array[$i]."<br/>"; } } get_urls("http://www.yinghy.com"); ?>
<?php $string = GetHtmlCode("http://www.yinghy.com"); echo $string; function GetHtmlCode($url){ $ch = curl_init();//初始化一个cur对象 curl_setopt ($ch, CURLOPT_URL, $url);//设置需要抓取的网页 curl_setopt ($ch, CURLOPT_RETURNTRANSFER, 1);//设置crul参数,要求结果保存到字符串中还是输出到屏幕上 curl_setopt ($ch, CURLOPT_CONNECTTIMEOUT,1000);//设置链接延迟 $HtmlCode = curl_exec($ch);//运行curl,请求网页 return $HtmlCode; } function GetAllLink($string) { $string = str_replace("\r","",$string); $string = str_replace("\n","",$string); $regex[url] = "((http|https|ftp|telnet|news):\/\/)?([a-z0-9_\-\/\.]+\.[][a-z0-9:;&#@=_~%\?\/\.\,\+\-]+)"; $regex[email] = "([a-z0-9_\-]+)@([a-z0-9_\-]+\.[a-z0-9\-\._\-]+)"; //去掉网页中的[] $string = eregi_replace("\[|\]","",$string); //去掉JAVASCRIPT代码 $string = eregi_replace("<!--.*//-->","", $string); //去掉非<a>的HTML标签 $string = eregi_replace("</?[^aA][^<>]*>","",$string); //分割$string中的所有链接 $output = split(\'</a>\', $string); for($i=0; $i<count($output); $i++){ $output_1 = split("<a", $output[$i]); } return $output_1; } function GetUserCareNews ($test,$keywords,$url) { $messTxt = ""; $k=0; $key = explode(";",$keywords); //自动为网站加载上http,避免网易邮箱链接错误,有一定的局限性 if(!ereg("http",$url)){ $url = "http://".$url; } for($i=0; $i<count($test); $i++){ $test[$i] = eval(\'return\'.iconv(\'gbk\',\'utf-8\',var_export($test[$i],true)).\';\');//修改编码 if(ereg("href", $test[$i]) && !ereg("href=\'#\'",$test[$i])){//去掉无效链接 for($j=0; $j<count($key); $j++){ //支持多关键字 if(strpos($test[$i],$key[$j])!==false){ $mess[$k++]=ereg_replace($key[$j],"<font color=red>".$key[$j]."</font>", $test[$i]);//高亮关键字 } } } } $mess = array_unique($mess); //数组去重 //处理好发送链接,为链接加上网站根目录 for($l=0; $l<count($mess); $l++){ if(!ereg("http",$mess[$l]) && (strlen($mess[$l]) != 0)){//去掉空数组,这步很重要,如果不去掉直接影响后面链接的质量 $mess[$l] = eregi_replace("href=[\"\']","",$mess[$l]); $mess[$l] = $url.$mess[$l]; $mess[$l] = eregi_replace(" /","/",$mess[$l]); if(ereg("\'",$mess[$l])){ $mess[$l]="<a href=\'".$mess[$l]."</a>"; } if(ereg("\"",$mess[$l])){ $mess[$l] = "<a href=\"".$mess[$l]."</a>"; } } else{ $mess[$l] = "<a ".$mess[$l]."</a>"; } $messTxt .= $mess[$l]; $messTxt .= "<BR>"; } return $messTxt; } function SendEmail($to, $content) { //Author:luofei //$to 表示收件人地址,$content表示邮件正文内容 error_reporting(E_STRICT); //错误报告 date_default_timezone_set("Asia/Shanghai"); //设定时区 require_once("class.phpmailer.php"); require_once("class.smtp.php"); $mail = new PHPMailer(); //新建一个对象 $mail->CharSet = "UTF-8"; //设置编码,中文不会出现乱码的情况 $mail->IsSMTP(); //设定使用SMTP服务 $mail->SMTPDebug = 1; //启用SMTP调试功能i //1 = errors and messages //2 = messages only $mail->SMTPSecure = "tls"; //安全协议 $mail->Host = "smtp.googlemail.com"; //SMTP服务器 $mail->SMTPAuth = true; //启用SMTP验证功能 $mail->Username = "[email protected]"; //SMTP服务器用户名 $mail->Password = "******"; //SMTP服务器用户密码 $mail->From = "[email protected]"; //发件人 $mail->FromName = "Spider Service"; //发件人姓名(邮件上显示) $mail->AddAddress($to); //收件人地址 $mail->WordWrap = 50; //设置邮件正文每行的字符数 $mail->IsHTML(true); //设置邮件正文内容是否为html类型 $mail->Subject = "来自spider.html的邮件"; //邮件主题 $mail->Body = "<p>您好!<BR> <p>这是您感兴趣的内容</p> <BR>".$content." "; //邮件正文 if(!$mail->Send()) //邮件发送报告 { echo "发送邮件错误!"; } else { echo "邮件发送成功!"; } } ?>
请发表评论