, and

tags and Meta information. ********* TITLE ************ The first 4 words in the tile get more points than the remaining words. The presence of these words in H1 tags accounts for most points, followed by their presence in first 100 text words, then H2-H6 tags, then the strong tags (, ,

) and finally the URL. **************************** ********* META TAGS ************ Google's crawler doesn't give a lot of importance to the meta keywords. This algorithm tries to do the same. The words in the meta tags keyowrds don't get a lot of points themselves but their presence in H1 tags, first 100 text words, H2-H6 tags, strong tags and the URL earns them extra points. ******************************** ********* H1 tags ************ The presence of the title and meta words in the H1 tags as well as other H1 words. **************************** ********* H2-H6 tags ************ The presence of the title and meta words in the H1 tags as well as other H1 words. **************************** Source : http://www.seomoz.org/article/search-ranking-factors#ranking-factors ***************************************************************************************************************************** *****************************************************************************************************************************/ //session_start(); $starttime = microtime(true); define('CURLTIMEOUT',5); define('HARDTIMEOUT',2); set_time_limit(0); ini_set('memory_limit', '2048M'); ini_set('xdebug.max_nesting_level', 100); require_once("crawlerVars.php"); require_once("crawlerGlobalFunctions.php"); require_once("simple_html_dom.php"); require_once("db.php"); ini_set("display_errors",1); error_reporting(1); $curl_options = array( CURLOPT_HEADER => false, CURLOPT_SSL_VERIFYPEER => false, CURLOPT_SSL_VERIFYHOST => false, CURLOPT_RETURNTRANSFER => true, CURLOPT_TIMEOUT => CURLTIMEOUT, CURLOPT_CONNECTTIMEOUT => 1, CURLOPT_FORBID_REUSE => true, CURLOPT_FRESH_CONNECT => true ); class mCache extends Memcache { static private $instance = null; static private $port = 11211; static function mGet() { if(self::$instance == null) { self::$instance = new Memcache; if(strstr($_SERVER['SERVER_ADDR'], "172.31.")) { self::$instance->addServer('172.31.6.33', self::$port); self::$instance->addServer('172.31.6.34', self::$port); self::$instance->addServer('172.31.6.36', self::$port); } else self::$instance->addServer('192.168.30.51', self::$port); } return self::$instance; } } class mCache_p extends Memcache { static private $instance = null; static private $port = 11211; static function mGet() { if (self::$instance == null) { self::$instance = new Memcache; if(strstr($_SERVER['SERVER_ADDR'], "172.31.")) { self::$instance->addServer('172.31.6.33', self::$port); self::$instance->addServer('172.31.6.34', self::$port); self::$instance->addServer('172.31.6.36', self::$port); } else { self::$instance->addServer("192.168.16.229", self::$port); self::$instance->addServer("192.168.16.230", self::$port); self::$instance->addServer("192.168.16.231", self::$port); self::$instance->addServer("192.168.16.232", self::$port); self::$instance->addServer("192.168.16.233", self::$port); } } return self::$instance; } } function getRefKeyword($ref) { $keyword = ""; $pKeyword = ""; // For Yahoo /////////////////////////////////////////////////////////////////////// //////////////////Search Re-targeting Code /////////////////////////// ///////////////////////////////////////////////////////////////////// $referrer = urldecode($ref); $refSp = explode("?", $referrer); $qString = ""; if(count($refSp) > 1) { $cn = count($refSp); $qString = $refSp[($cn-1)]; } else if(count($refSp) == 1) $qString = $referrer; $qStringSp = explode("#", $qString); if(count($qStringSp) > 1 && trim($qStringSp[1]) != '') { $qString = $qStringSp[2]; } $params = explode("&", $qString); foreach($params as $val) { $val = trim($val); $keyVal = explode("=", $val); if($keyVal[0] == "q") { $keyword = trim($keyVal[1]); $keyword = str_replace("+", " ", $keyword); } if($keyVal[0] == "p") { $pKeyword = trim($keyVal[1]); $pKeyword = str_replace("+", " ", $pKeyword); } } if(trim($keyword) == '' && trim($pKeyword) != '') $keyword = $pKeyword; $keyword = str_replace("%20", " ", $keyword); return $keyword; //////////////////////////////////////////////////////////////////////////////////////// } $callback = "handleLoading"; if(isset($_GET['callback']) && trim($_GET['callback']) != '') $callback = $_GET['callback']; //$arr = array("192.168.16.16","192.168.16.17","192.168.16.18","192.168.16.19"); if(isset($argv[1])){ $arr = array(); $arr[] = $argv[1]; } if(isset($argv[2])){ $alphabet = $argv[2]; } if($alphabet){ $table_structure = "crawler_urls_".$alphabet; } else { $table_structure = 'crawler_urls_%'; } $tot_count = 0; for($loop=0; $loop<500; $loop++){ $rand_number = rand(1,2000); $startprocess = microtime(true); foreach($arr as $server){ $e = microtime(true)-$starttime ; // print "Processing $server...[".number_format($e,2)."]\n "; $db = ConnectToIP($server); $sql = " show tables like '$table_structure' "; $res1 = mysql_query($sql, $db); while($row1=mysql_fetch_array($res1)){ $feeds = array(); $tablename = $row1[0]; $db1 = ConnectToIP($server); if(rand(1,10) <2){ $sql = " select * from Shorty.$tablename where processed=0 and priority=1 limit 2000; "; } else { $sql = " select * from Shorty.$tablename where processed=0 limit $rand_number, 10; "; } $res = mysql_query($sql, $db1); while($row=mysql_fetch_assoc($res)){ $feeds[] = $row['url']; $tot_count++; } $curlstart = microtime(true); $response = syncRequest($feeds); $curlstop = microtime(true); $str = ""; $e = microtime(true)-$starttime ; // print "Response time[$e]\n"; // print_r($feeds); // print_r($response); // die(); //foreach($feeds as $key => $value) { // echo "\n[$key] = $value"; //} //// //echo "\n\nResults::::\n"; //foreach($response as $key => $value) { // echo "\n[$key] = ".strlen($value); //} // //die("done"); for($j=0; $j0){ crawler($url, "i", $r, "", "", $tablename, $server, $content); } else { $db_conn = ConnectToIP($server); $insert_sql = " update Shorty.$tablename set content='', processed=1, meta_keywords='', timedate =unix_timestamp() where url='$url' limit 1; "; mysql_query($insert_sql,$db_conn); mysql_close($db_conn); } $str .= "[Length:".strlen($content)."][".number_format(microtime(true)-$s, 2)."]\n" ; // print "[Count:$j [$url] : Length:".strlen($content)."][".number_format(microtime(true)-$s, 2)."]\n"; } // print $str."\n"; mysql_close($db1); } mysql_close($db); $elapsed = microtime(true)-$starttime ; if($elapsed > 850){ print "1: Cron finished in $elapsed seconds [count= $tot_count] [$server][$tablename] at ".date("Y-m-d H:i:s")."\n"; die(); } } $elapsed = microtime(true)-$starttime ; if($elapsed > 850){ print "2: Cron finished in $elapsed seconds [count= $tot_count] [$server][$tablename] at ".date("Y-m-d H:i:s")."\n"; die(); } } function crawler($link, $type, $referrer, $keyword, $affiliate, $tablename, $server, $content) { global $inlineKeys, $insertKeys, $contextualBids, $dbServers, $callback; /******** Get The current page URL *****************/ if(!isset($link) || $link == '') { $cr_protocol = strpos(strtolower($_SERVER['SERVER_PROTOCOL']),'https') === FALSE ? 'http' : 'https'; $cr_host = $_SERVER['HTTP_HOST']; $cr_script = $_SERVER['SCRIPT_NAME']; $cr_params = $_SERVER['QUERY_STRING']; $currentUrl = $cr_protocol . '://' . $cr_host . $cr_script . '?' . $cr_params; } else $currentUrl = $link; $crawl = 1; /////////////////////////////////////////////////////////////////////// if($crawl == 1) { doCrawl_new($content, $currentUrl, $affiliate, "i", $link, $tablename, $server); } if($keyword != '') $retArr['contextual'] = $keyword; else $retArr['contextual'] = $insertKeys; $retArr['contextualBids'] = $contextualBids; header("Access-Control-Allow-Origin: *"); if($type == "i") { $retArr['inline'] = $inlineKeys; // echo "{$callback}(" . json_encode($retArr) .");"; } else echo $retArr['contextual']; } function doCrawl_new($html, $webpage, $affiliate, $cType = "i", $url, $tablename, $server) { if(strstr($webpage, "bentrending.com")) { $html = str_replace("<", "<", $html); $html = str_replace(">", ">", $html); } if(strstr($webpage, "cellphonedigest.net") || strstr($webpage, "myinteriordecorator.com") || strstr($webpage, "webtrading.com") || strstr($webpage, "accoona.com") || strstr($webpage, "menopause-weight-gain-help.com")) $html = stripText($html); if(strstr($webpage, "travlang.com") || strstr($webpage, "southdeliciousfood.blogspot.com")) $html = substr($html, 0, 20000); global $vars, $fp, $htmlDOM; global $keywordsArr, $allKeyWords, $n; global $h1Words, $textOnly, $strongWords, $h2_h6Words, $linkTextWords, $imgAltTextWords, $domainWords, $first300, $pWords; global $insertKeys, $inlineKeys, $contextualBids, $textMinLimit; global $dbServers; global $commonWords; //unset($keywordsArr); $insertKeys = ""; $inlineKeys = ""; $keywordsArr = array(); $allKeyWords = ""; //echo "{$n}. {$webpage}
"; $html_ok = 1; $htmlDOM = str_get_html($html); $webhost = ""; /////////////////////////Get Page Text ///////////////////////////////////////////////////// $pageTextHTML = ""; $pageTextHTML = getPageText($html, $htmlDOM, $webhost); $pageTextHTML = strtolower($pageTextHTML); $advMatch_kwords = array(); $keyword_tag = ""; $desc_tag = ""; if($html_ok) $tags = getMetaInfo($html); if(!$tags['keywords'] || trim($tags['keywords']) == '') $tags = get_meta_tags($webpage); if($tags['keywords'] && $tags['keywords'] != '') $keyword_tag = strtolower($tags['keywords']); //echo $tags['keywords']; $tags = getMetaDescInfo($html); $desc_tag = strtolower($tags['description']); if($keyword_tag != '') { $keyword_tag_ex = explode(",", $keyword_tag); foreach($keyword_tag_ex as $kt) { $kt = trim($kt); if(str_word_count($kt) > 3) { $kt_ex = explode(" ", $kt); foreach($kt_ex as $kex) { $keywordsArr[] = $kex; } } else $keywordsArr[] = $kt; } } $contextDesc = ""; if($desc_tag != '') { $contextDesc = ignoreChars($desc_tag); $contextDesc = ignoreWords($contextDesc); $contextDesc = removeMultiSpaces($contextDesc); } ///////////////////Get Words from the H1 Tags ////////////////////////////////////////////// $h1Words = ""; if($html_ok) { //$h1Words = getH1Tags($htmlDOM); } ///////////////////Get Words from the H2 - H6 Tags ///////////////////////////////////////// $h2_h6Words = ""; if($html_ok) { $h2_h6Words = getH2_H6Tags($htmlDOM); } //////////////////////////////////////////////////////////////////////////////////////////// $contextHTML = $contextDesc." ".$h2_h6Words; $contextHTML_ex = explode(" ", $contextHTML); foreach($contextHTML_ex as $cex) { $cex = trim($cex); $keywordsArr[] = $cex; } //die(var_dump($keywordsArr)); /**************************************************************************************************************** ** Crawling process ends * ****************************************************************************************************************/ $i = 0; $con = 0; $inl = 0; $insertKeys = ""; $inlineArr = array(); $occArr = array(); foreach($keywordsArr as $k) { if(strlen($k) > 2 && !is_numeric($k)) { $k = trim($k); if($cType == "i" && strpos($pageTextHTML, " ".$k." ") !== false && str_word_count($k) <= 3) { $inlineKeys .= $k; $inlineKeys .= ","; $inlineArr[] = $k; $inl++; } $i++; } } ///////////////////////////////////////////////////////// if($cType == "i") { $newInline = array(); $inlineKeys = ""; $skipV = 0; $test = ""; $singleWords = 0; foreach($inlineArr as $v) { $skipV = 0; if(trim($v) != '') { $no = 0; foreach($inlineArr as $r) { if(trim($r) != '') { //$test .= $r.","; $n = 0; $nWord = $v." ".$r; //$nWordCh = preg_replace("/([,.?!])/"," \\1",$nWord); $nWordSp = explode(" ",$nWord); $nWordSp = array_unique($nWordSp); $nWord = implode(" ",$nWordSp); if(strpos($pageTextHTML, $nWord) !== false && substr_count($nWord, " ") > 0) { if(!in_array($nWord, $newInline) && str_word_count($nWord) <= 3) { $newInline[] = $nWord; //$skipV = 1; } //$inlineKeys .= $nWord; //$inlineKeys .= ","; } } } if($skipV == 0) { if(!in_array($v, $newInline)) $newInline[] = $v; } } } $db_conn = ConnectToIP($server); $insert_sql = " update Shorty.$tablename set content='".mysql_real_escape_string($pageTextHTML)."', processed=1, meta_keywords='".implode(",",$newInline)."', timedate =unix_timestamp() where url='$url' limit 1; "; // print $insert_sql; mysql_query($insert_sql,$db_conn); mysql_close($db_conn); // print "Processed $url [$tablename] [$server]
"; } $n++; } function syncRequest($data) { global $curlstart, $timedout, $curl_options; $curly = array(); $result = array(); $mh = curl_multi_init(); foreach ($data as $id => $url) { $curly[$id] = curl_init($url); curl_setopt_array($curly[$id], $curl_options); curl_multi_add_handle($mh, $curly[$id]); } $running = null; $hard_timeout_reached = false; $do_count = 0; do { if((microtime(true) - $curlstart) > HARDTIMEOUT) { $timedout++; $hard_timeout_reached = true; break; } curl_multi_exec($mh, $running); $do_count ++; } while($running > 0 && $do_count < 30000); foreach($curly as $id => $c) { $result[$id] = curl_multi_getcontent($c); curl_multi_remove_handle($mh, $c); } curl_multi_close($mh); return $result; } $elapsed = microtime(true)-$startprocess ; print "Cron finished in $elapsed seconds [count= $tot_count] [$server][$tablename] at ".date("Y-m-d H:i:s")."\n"; ?>