<?php
//	Author:	Kushal Khatri
/****************************************************************************************************************************
************************************CRAWLING ALGORITHM **********************************************************************
*****************************************************************************************************************************

This algorithm takes into consideration title, domain name, H1 tags, text, H2-H6 tags, <b>, <strong> and <li> tags and Meta information.

********* TITLE ************
The first 4 words in the tile get more points than the remaining words. The presence of these words in H1 tags accounts for 
most points, followed by their presence in first 100 text words, then H2-H6 tags, then the strong tags (<b>, <strong>, <li>)
and finally the URL.
****************************

********* META TAGS ************
Google's crawler doesn't give a lot of importance to the meta keywords. This algorithm tries to do the same.

The words in the meta tags keyowrds don't get a lot of points themselves but their presence in H1 tags, first 100 text words, 
H2-H6 tags, strong tags and the URL earns them extra points. 
********************************

********* H1 tags ************
The presence of the title and meta words in the H1 tags as well as other H1 words.
****************************

********* H2-H6 tags ************
The presence of the title and meta words in the H1 tags as well as other H1 words.
****************************

Source : http://www.seomoz.org/article/search-ranking-factors#ranking-factors

*****************************************************************************************************************************
*****************************************************************************************************************************/
//session_start();
require_once("crawlerVars.php");
require_once("crawlerFunctions.php");
require_once("simple_html_dom.php");

if($_GET['debug'] == "ms123")
	echo $_SERVER['SERVER_ADDR']." - ".$_SERVER['HTTP_HOST']."<br>";

class mCache extends Memcache {
	static private $instance = null;
	static private $port = 11211;
	static function mGet() {
		if(self::$instance == null) {
			self::$instance = new Memcache;
			self::$instance->addServer('192.168.16.191', self::$port);
			self::$instance->addServer('192.168.16.216', self::$port);
			self::$instance->addServer('192.168.16.242', self::$port);
		}
		return self::$instance;
	}
}

class mCache_p extends Memcache {
    static private $instance = null;
    static private $port = 11211;
    static function mGet() {
        if (self::$instance == null) {
            self::$instance = new Memcache;
			self::$instance->addServer("192.168.16.223", self::$port);
			self::$instance->addServer("192.168.16.224", self::$port);
			self::$instance->addServer("192.168.16.225", self::$port);
			self::$instance->addServer("192.168.16.226", self::$port);
			self::$instance->addServer("192.168.16.227", self::$port);
        }
        return self::$instance;
    }
}


function getRefKeyword($ref)
{
		$keyword = "";
		$pKeyword = ""; // For Yahoo
		///////////////////////////////////////////////////////////////////////
		//////////////////Search Re-targeting Code ///////////////////////////
		/////////////////////////////////////////////////////////////////////
		$referrer = urldecode($ref);
		
		$refSp = explode("?", $referrer);
		$qString = "";
		if(count($refSp) > 1)
		{
			$cn = count($refSp);
			$qString = $refSp[($cn-1)];
		}
		else if(count($refSp) == 1)
			$qString = $referrer;
		
		$qStringSp = explode("#", $qString);
		if(count($qStringSp) > 1 && trim($qStringSp[1]) != '')
		{
			$qString = $qStringSp[2];
		}
		
		$params = explode("&", $qString);
		foreach($params as $val)
		{
			$val = trim($val);
			$keyVal = explode("=", $val);
			if($keyVal[0] == "q")
			{
				$keyword = trim($keyVal[1]);
				$keyword = str_replace("+", " ", $keyword);
			}
			if($keyVal[0] == "p")
			{
				$pKeyword = trim($keyVal[1]);
				$pKeyword = str_replace("+", " ", $pKeyword);
			}
		}
		if(trim($keyword) == '' && trim($pKeyword) != '')
			$keyword = $pKeyword;
		$keyword = str_replace("%20", " ", $keyword);
		return $keyword;
		////////////////////////////////////////////////////////////////////////////////////////
}

// Track Impresssions
function connectToShServ($ip)
{
	$mysql_host = $ip; 
	$mysql_user = "c3slave";
	$mysql_password = "RstG496urky2";
	$mysql_db = "Shorty";
	$dbd = mysql_connect ($mysql_host, $mysql_user, $mysql_password );
	@mysql_select_db ($mysql_db, $dbd);
	return $dbd; 
}

$shServers = array("192.168.16.4", "192.168.16.7", /*"192.168.16.8",*/ "192.168.16.9");
shuffle($shServers);
$shServ = $shServers[0];
$shdb = connectToShServ($shServ);

$ad_table = "crawler_impr";
$tdate = date("Y-m-d");
$insert_sql = " insert into $ad_table set
		date='$tdate',
		count = 1
		on duplicate key update 
		count=count+1";

$res= mysql_query($insert_sql,$shdb);
if (!$res){
	$errno = mysql_errno();
	$errtx = mysql_error();
	if ($errno==1146){
		$create_table = "
			CREATE TABLE if not exists $ad_table (
			date DATE NOT NULL PRIMARY KEY,
			count int(11) not null default 1
		)";
		$res1 = mysql_query($create_table,$shdb);
	}
}

mysql_close($shdb);
//////////////////////////////

$callback = "handleLoading";
if(isset($_GET['callback']) && trim($_GET['callback']) != '')
	$callback = $_GET['callback'];

$affiliate = "";
if(isset($_GET['affiliate']) && trim($_GET['affiliate']) != '')
	$affiliate = $_GET['affiliate'];

if($affiliate == '' && isset($_GET['affid']) && trim($_GET['affid']) != '')
	$affiliate = $_GET['affid'];

if((!isset($_GET['l']) || trim($_GET['l']) == '') && trim($_GET['r']) != '')
{
	$keyword = getRefKeyword($_GET['r']);
	echo $keyword;
}

if(isset($_GET['l']) && $_GET['l'] != '')
{
	$r = "";
	if(isset($_GET['r']))
		$r = $_GET['r'];
	$retArr = array();
	$keyword = "";
	if(trim($r) != '')
	{
		$keyword = getRefKeyword($r);
	}

	if(trim($keyword) == '')
		crawler($_GET['l'], $_GET['type'], $r, '', $affiliate);
	else
	{
		if($_GET['type'] == "i")
			crawler($_GET['l'], $_GET['type'], $r, $keyword, $affiliate);
		else
		{
			//$retArr['contextual'] = $keyword;
			//$retArr['inline'] = "";
			echo $keyword;
		}
	}
}

function crawler($link, $type, $referrer, $keyword, $affiliate)
{
global $inlineKeys, $insertKeys, $contextualBids, $dbServers, $callback;

/*$randServ = array_rand($dbServers);
if(!$randServ)
	$randServ=0;

$useServ = $dbServers[$randServ];

$cr_conn = mysql_connect($useServ,"web","RstG496urky2");
$cr_db = mysql_select_db("Live", $cr_conn);
*/
//$cr_conn = ConnectToRead();
/******** Get The current page URL *****************/

if(!isset($link) || $link == '')
{
$cr_protocol = strpos(strtolower($_SERVER['SERVER_PROTOCOL']),'https') 
                === FALSE ? 'http' : 'https';
$cr_host     = $_SERVER['HTTP_HOST'];
$cr_script   = $_SERVER['SCRIPT_NAME'];
$cr_params   = $_SERVER['QUERY_STRING'];
 
$currentUrl = $cr_protocol . '://' . $cr_host . $cr_script . '?' . $cr_params;
}
else
	$currentUrl = $link;

$crawl = 1;
$expire = "604800";
$mkey = md5("crawler_".$currentUrl);
if ($crawled_array = mCache::mGet()->get($mkey)){
	$insertKeys = $crawled_array['contextual'];
	$inlineKeys = $crawled_array['inline'];
	if($crawled_array['contextualBids'])
		$contextualBids = $crawled_array['contextualBids'];
	if($type != "i" && trim($insertKeys) != '')
		$crawl = 0;
	else if($type == "i" && trim($inlineKeys) != '')
		$crawl = 0;
} 

//mysql_close($cr_conn);
$retArr = array();
if($_GET['debug'] == "ms123")
	$crawl = 1;

///////////////////////////////////////////////////////////////////////
if($crawl == 1)
{
	if(strpos($currentUrl, "banner.php") === false)
	{
		// Insert URLs for alternate crawler
		$insUrl = 0;
		$priority = 0;

		$expire = "259200";
		$mpkey = md5("crwlerCntr2024_".$currentUrl);
		if($crawlerCntVal = mCache_p::mGet()->get($mpkey)){
			$crawlerCntVal_ex = explode("::", $crawlerCntVal);
			$crawlerCount = (int)$crawlerCntVal_ex[0];
			$crawlerTs = $crawlerCntVal_ex[1];

			$expire = $crawlerTs - time();
			$crawlerCount += 1;
			$crawlerCntNewVal = $crawlerCount."::".$crawlerTs;

			if($_GET['debug'] == "ms123")
				echo "MC Crawl Count: ".$crawlerCount."<br>";

			if($crawlerCount %20 == 0)
				$insUrl = 1;

			if($crawlerCount > 100)
				$priority = 1;
			else
				mCache_p::mGet()->replace($mpkey, $crawlerCntNewVal, 0, $expire);
		}
		else
		{
			$crawlerEx = time()+259200;
			$crawlerCntVal = "1::".$crawlerEx;
			mCache_p::mGet()->set($mpkey, $crawlerCntVal, 0, $expire);
			if($_GET['debug'] == "ms123")
				echo "New MC Count Ins<br>";
		}

		if($insUrl == 1)
		{
			$urlsArr = array('192.168.16.16','192.168.16.17','192.168.16.18','192.168.16.19');

			$currentDomain = preg_replace("/^(http(s)?:\/\/)*(www.)*/is", "", strtolower($currentUrl)); 
			$currentDomain = preg_replace("/\/.*$/is" , "" ,$currentDomain); 
			$domainAlpha = substr($currentDomain, 0, 1);
			if(!ctype_alpha($domainAlpha))
			{
				$domainAlpha = "0";
				$crHost = $urlsArr[0];
			}

			if(in_array($domainAlpha, range('a', 'f')))
				$crHost = $urlsArr[0];
			else if(in_array($domainAlpha, range('g', 'l')))
				$crHost = $urlsArr[1];
			else if(in_array($domainAlpha, range('m', 's')))
				$crHost = $urlsArr[2];
			else
				$crHost = $urlsArr[3];

			$sdb = mysql_connect($crHost, 'c3slave', 'RstG496urky2');
			mysql_select_db('Shorty', $sdb);

			$tname = "crawler_urls_".$domainAlpha;
			$iSql = "insert into {$tname} (url, priority) values ('{$currentUrl}', {$priority}) on DUPLICATE KEY Update count=count+1, priority = {$priority}";
			$iRes = mysql_query($iSql, $sdb);
			if(!$iRes)
			{
				$errno = mysql_errno();
				if ($errno==1146){
					$create_table = "
						CREATE TABLE IF NOT EXISTS {$tname} (
						  `url` varchar(200) NOT NULL,
						  `keycount` int(5) NOT NULL,
						  `count` int(11) NOT NULL DEFAULT '0',
						  `content` text,
						  `processed` tinyint(1) NOT NULL DEFAULT '0',
						  `timedate` int(11) NOT NULL DEFAULT '0',
						  `keywords` varchar(500) NOT NULL,
						  `meta_keywords` varchar(500) NOT NULL,
						  `priority` tinyint(1) NOT NULL DEFAULT '0',
						  PRIMARY KEY (`url`),
						  FULLTEXT KEY `ft_index` (`content`)
					) ENGINE=MyISAM DEFAULT CHARSET=latin1;";
					$res1 = mysql_query($create_table,$sdb) ;
					$res1= mysql_query($iSql,$sdb) ;
				}
			}

			mysql_close($sdb);
		}
	}

	if($_GET['debug'] == "ms123")
		echo "Crawl<br>";
	$currentUrl = str_replace(" ", "%20", $currentUrl);
	$cURL = curl_init();
	curl_setopt($cURL, CURLOPT_URL, $currentUrl);
	curl_setopt($cURL, CURLOPT_HEADER, 0);
	curl_setopt($cURL, CURLOPT_RETURNTRANSFER, 1);
	curl_setopt($cURL, CURLOPT_USERAGENT, "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.96 Safari/537.36");
	curl_setopt($cURL, CURLOPT_TIMEOUT, 30);
	curl_setopt($cURL, CURLOPT_SSL_VERIFYPEER, false);
	curl_setopt($cURL, CURLOPT_FOLLOWLOCATION, true);
	$cData = curl_exec($cURL); 
	//echo $cData;
	if(!curl_errno($cURL) && trim($cData) != '')
	{
		doCrawl($cData, $currentUrl, $affiliate, "i");
	}
	else
	{
		//$uSql = "update context_words set active = 0 where url='{$currentUrl}'";
		//$uRes = mysql_query($uSql);
	}
	curl_close($cURL);
	$memArr = array();
	$memArr['contextual'] = $insertKeys;
	$memArr['inline'] = $inlineKeys;
	$memArr['contextualBids'] = $contextualBids;
	mCache::mGet()->set($mkey, $memArr, 0, $expire);
	if($_GET['debug'] == "ms123")
		echo $mkey."<br>";
}

if($keyword != '')
	$retArr['contextual'] = $keyword;
else
	$retArr['contextual'] = $insertKeys;

$retArr['contextualBids'] = $contextualBids;
/*if($affiliate == "sharon_products")
{
	$retArr_ex = explode(",", $retArr['contextual']);
	shuffle($retArr_ex);
	if(trim($retArr_ex[0]) != '')
	{	
		$relWord = str_replace(" ", "+", $retArr_ex[0]);
		$cr_conn = ConnectToRead();
		$retSql = "select relatedWords from rep_relatedWords where keyword = '{$relWord}'";
		$retRes = mysql_query($retSql);
		if($retRes)
		{
			$retRow = mysql_fetch_assoc($retRes);
			if($retRow && trim($retRow['relatedWords']) != '')
			{
				$relatedWords = explode(",", $retRow['relatedWords']);
				$retArr['contextual'] = trim($relatedWords[0]);
			}
		}
		mysql_close($cr_conn);
	}
}*/

header("Access-Control-Allow-Origin: *");
if($type == "i")
{
	$retArr['inline'] = $inlineKeys;
	echo "{$callback}(" . json_encode($retArr) .");";
}
else
	echo $retArr['contextual'];
}
?>