<?php

function doCrawl($html, $webpage, $affiliate, $cType = "i")
{	
	$stime = microtime(true);
	if(strstr($webpage, "bentrending.com"))
	{
		$html = str_replace("&lt;", "<", $html);
		$html = str_replace("&gt;", ">", $html);
	}

	if(strstr($webpage, "cellphonedigest.net") || strstr($webpage, "myinteriordecorator.com") || strstr($webpage, "webtrading.com") || strstr($webpage, "accoona.com") || strstr($webpage, "menopause-weight-gain-help.com"))
		$html = stripText($html);
	
	if(strstr($webpage, "travlang.com") || strstr($webpage, "southdeliciousfood.blogspot.com"))
		$html = substr($html, 0, 20000);
	global $vars, $fp, $htmlDOM;
	global $keywordsArr, $allKeyWords, $n;
	global $h1Words, $textOnly, $strongWords, $h2_h6Words, $linkTextWords, $imgAltTextWords, $domainWords, $first300, $pWords;
	global $insertKeys, $inlineKeys, $inlineMetaKeys, $contextualBids, $textMinLimit;
	global $dbServers;
	global $commonWords;
	//unset($keywordsArr);
	$insertKeys = "";
	$inlineKeys = "";
	$inlineMetaKeys = "";

	$keywordsArr = array();
	$allKeyWords = "";
	//echo "{$n}. <a href='{$webpage}' target='_blank'>{$webpage}</a><br>";
	$html_ok = 1;
	
	$htmlDOM = str_get_html($html);
	
	$weburl = "";
	
	/////////////////////////Get Page Text /////////////////////////////////////////////////////
	$pageTextHTML = "";
	$pageTextHTML = getPageText($html, $htmlDOM, $webhost);
	$pageTextHTML = strtolower($pageTextHTML);
	//$pageTextHTML = ignoreChars($pageTextHTML);
	
	$advMatch_kwords = array();

	//echo substr($pageTextHTML, 0, 20000);
	//echo strlen($pageTextHTML);
	/*$pageTextHTML = str_replace('"', '', $pageTextHTML);
	$pageTextHTML = str_replace("'", "", $pageTextHTML);

	die($pageTextHTML);*/

	if($_GET['advKeywords'] == "adv" && (isset($_GET['aid']) && $_GET['aid'] == "41997"))
	{
		$wm = 15000;
		if(isset($_GET['wm']) && $_GET['wm'] != '')
			$wm = $_GET['wm'];
		if($_GET['debug'] == "ms123")
		{
			echo "Start adv fn<br>";
			echo substr($pageTextHTML, 0, $wm)."<br>";
		}
		$start_time = time();
		$advMatch_kwords = getAdvKeywords(substr($pageTextHTML, 0, $wm));
		$end_time = time();
		if($_GET['debug'] == "ms123")
		{
			echo "Time : ".($end_time - $start_time)." secs<br>";
			echo "Advertiser Keywords:: ";
			var_dump($advMatch_kwords);
			echo "<br>";
		}
	}
	/////////////////////////////////////////////////////////////////////////////////////////////
	
	$keyword_tag = "";
	$desc_tag = "";
	
	if($html_ok)
		$tags = getMetaInfo($html);
	
	if(!$tags['keywords'] || trim($tags['keywords']) == '')
		$tags = get_meta_tags($webpage);
	
	if($tags['keywords'] && $tags['keywords'] != '')
		$keyword_tag = strtolower($tags['keywords']);
	//echo $tags['keywords'];
	if(!$tags['description'] || $tags['description'] == '')
		$tags = getMetaDescInfo($html);
	$desc_tag = strtolower($tags['description']);
	
	if($keyword_tag != '')
	{
		$keyword_tag_ex = explode(",", $keyword_tag);
		foreach($keyword_tag_ex as $kt)
		{
			$kt = trim($kt);
			if(str_word_count($kt) > 3)
			{
				$kt_ex = explode(" ", $kt);
				foreach($kt_ex as $kex)
				{
					if(strlen($kex) >= 4)
						$keywordsArr[] = $kex;
				}
			}
			else if(strlen($kt) >= 4)
				$keywordsArr[] = $kt;
		}
	}
	$contextDesc = "";
	if($desc_tag != '')
	{
		$contextDesc = ignoreChars($desc_tag);
		$contextDesc = ignoreWords($contextDesc);
		$contextDesc = removeMultiSpaces($contextDesc);
	}

	///////////////////Get Words from the H1 Tags //////////////////////////////////////////////
	
	$h1Words = "";
	if($html_ok)
	{
		//$h1Words = getH1Tags($htmlDOM);
	}
	
	///////////////////Get Words from the H2 - H6 Tags /////////////////////////////////////////
	
	$h2_h6Words = "";
	if($html_ok)
	{
		$h2_h6Words = getH2_H6Tags($htmlDOM);
	}
	////////////////////////////////////////////////////////////////////////////////////////////

	$contextHTML = $contextDesc." ".$h2_h6Words;
	$contextHTML_ex = explode(" ", $contextHTML);
	foreach($contextHTML_ex as $cex)
	{
		$cex = trim($cex);
		if(strlen($cex) >= 4)
			$keywordsArr[] = $cex;
	}

	//die(var_dump($keywordsArr));
	/****************************************************************************************************************
	**							Crawling process ends																*
	****************************************************************************************************************/

	$i = 0;
	$con = 0;
	$inl = 0;
	$insertKeys = "";
	$inlineArr = array();
	$occArr = array();
	
	foreach($keywordsArr as $k)
	{	//echo $k."<br>";
		if(strlen($k) > 2 && !is_numeric($k))
		{
			$k = trim($k);
			
			if($cType == "i" && strpos($pageTextHTML, " ".$k." ") !== false && str_word_count($k) <= 3 && !in_array($k, $inlineArr))
			{	
				$inlineArr[] = $k;
				$inl++;
			}
			$i++;
		}	

		if($inl > 10)
			break;
	}
	
	/////////////////////////////////////////////////////////

	if($cType == "i")
	{
		$newInline = array();
		$inlineKeys = "";
		$skipV = 0;
		$test = "";
		$singleWords = 0;

		foreach($inlineArr as $v)
		{
			$skipV = 0;
			if(trim($v) != '')
			{
				$no = 0;
				foreach($inlineArr as $r)
				{
					if(trim($r) != '')
					{
						//$test .= $r.",";
						$n = 0;
						$nWord = $v." ".$r;
						$nWordCh = preg_replace("/([,.?!])/"," \\1",$nWord);
						$nWordSp = explode(" ",$nWordCh);
						$nWordSp = array_unique($nWordSp);
						$nWord = implode(" ",$nWordSp);
						if(strpos($pageTextHTML, " ".$nWord." ") !== false && substr_count($nWord, " ") > 0)
						{	
							if(!in_array($nWord, $newInline) && str_word_count($nWord) <= 3)
							{
								$newInline[] = $nWord;
								//$skipV = 1;
							}
							//$inlineKeys .= $nWord;
							//$inlineKeys .= ",";
						}
					}
				}
				if($skipV == 0)
				{
					if(!in_array($v, $newInline))
						$newInline[] = $v;
				}
			}
		}
		
		//$inlineKeys = implode(",", $inlineArr);
		if(count($advMatch_kwords) > 0)
		{
			$inlineKeys = implode(",", $advMatch_kwords);
		}
		$inlineMetaKeys = implode(",", $newInline);
	}
	//echo round((microtime(true) - $stime), 4)." secs<br>";
	$n++;
}

function getAdvKeywords($pageTextHTML)
{
	$wordsToMatch = 6;
	if(isset($_GET['wtm']))
		$wordsToMatch = $_GET['wtm'];

	$stime = microtime(true);
	if($_GET['debug'] == "ms123")
		echo "Entered getAdvKeywords<br>";

	$matched_keywords = array();
	$adv_keywords = array();
	$memForBreak = 0;

	$keyPr = "crawler_advKeywords__690__ebnl4k_";
	for($i=1; $i<=1000; $i++)
	{
		$chStime = microtime(true);
		$slArr = array();
		$mkey = md5($keyPr.$i);
		$slArr = mCache_p::mGet()->get($mkey);
		$chEtime = microtime(true);

		if($_GET['debug2'] == "ms123")
				echo $i." - ".round(($chEtime - $chStime), 3)." secs memcache time - ".count($slArr)." words :: ";
		if($slArr && is_array($slArr))
		{
			//$adv_keywords = array_merge($adv_keywords, $slArr);
			foreach($slArr as $keyword=>$v)
			{
				$keyword = preg_replace("/[^A-Za-z0-9 '\s-_]/", "", $keyword);
				if($keyword != '' && !is_numeric($keyword) && strpos($pageTextHTML, " ".$keyword." ") !== false)
				{
					$matched_keywords[] = $keyword;
					if(count($matched_keywords) == $wordsToMatch)
					{
						$memForBreak = 1;
						break;
					}
				}
			}
		}

		if($_GET['debug2'] == "ms123")
			echo count($matched_keywords)." matched words :: ".round((microtime(true) - $chEtime), 3)." secs match time<br>";

		if($memForBreak == 1)
			break;
	}

	return $matched_keywords;
}

function sortByLength($a,$b){
  if(str_word_count($a) == str_word_count($b)) return 0;
  return (str_word_count($a) > str_word_count($b) ? -1 : 1);
}

function getTitlePoints($word, $isDouble=0)
{
	global $vars;
	global $h1Words, $domainWords, $strongWords, $h2_h6Words, $imgAltTextWords, $textOnly, $linkTextWords, $pWords;

	$pts = 0;
	$h1occ = 0;
	$docc = 0;
	$stocc = 0;
	$h2_h6occ = 0;
	$aocc = 0;
	$ialtocc = 0;
	$tocc = 0;
		
	$h1occ = substr_count($h1Words, $word);
	//$focc = substr_count($first100Words, $doubleWord);
	$docc = substr_count($domainWords, $word);
	$stocc = substr_count($strongWords, $word);
	$h2_h6occ = substr_count($h2_h6Words, $word);
	$ialtocc = substr_count($imgAltTextWords, $word);
	$tocc = 0;
	$aocc = substr_count($linkTextWords, $word);
	$pocc = substr_count($pWords, $word);
	//$tocc -= $pocc;

	if($isDouble == 1)
	{
		$pts = ($vars['title_h1Factor'] * $h1occ) + ($vars['title_domFactor'] * $docc) + ($vars['title_stFactor'] * $stocc) + ($vars['title_h2h6Factor'] * $h2_h6occ) + ($vars['title_ialtFactor'] * $ialtocc) + ($vars['title_aFactor'] * $aocc) + ($vars['title_pFactor'] * $pocc);
		
	}
	else
	{
		$pts += ($vars['title_h1Factor'] * $h1occ) + ($vars['title_domFactor'] * $docc) + ($vars['title_stFactor'] * $stocc) + ($vars['title_h2h6Factor'] * $h2_h6occ) + ($vars['title_aFactor'] * $aocc) + ($vars['title_ialtFactor'] * $ialtocc) + ($vars['title_tFactor'] * $tocc) + ($vars['title_pFactor'] * $pocc);
	}
	return $pts;
}

function implode_key($glue = " ", $pieces = array()) 
{ 
    $arrK = array_keys($pieces); 
    return implode($glue, $arrK); 
} 

function getWordsFromString($string, $mode)
{
	global $keywordsArr, $allKeyWords;
	global $h1Words, $strongWords, $domainWords, $h2_h6Words, $linkTextWords, $textOnly, $imgAltTextWords, $pWords;
	global $vars;

	//echo "{$h1Words}<br><br>{$strongWords}<br><br>{$domainWords}<br><br>{$h2_h6Words}";
  if($mode == "meta")
  {
	$lastWord = "";
	$titleWords = explode(" ", $string);
	foreach($titleWords as $k=>$v)
	{
		$skip = 0;
		$v = trim($v);
		$doubleWord = "";
		if(trim($lastWord) != '')
			$doubleWord = $lastWord." ".$v;
		
		$pts = 0;
		
		//$allKeyWords = implode_key(" ",$keywordsArr);
		if(preg_match("/{$v}/",$allKeyWords) == 1)
			$skip = 1;
		if($skip != 1)
		{
			$h1occ = 0;
			$docc = 0;
			$stocc = 0;
			$h2_h6occ = 0;
			$aocc = 0;
			$ialtocc = 0;
			$tocc = 0;
			$pocc = 0;

			$h1occ = substr_count($h1Words, $v);
			//$focc = substr_count($first100Words, $v);
			$stocc = substr_count($strongWords, $v);
			$docc = substr_count($domainWords, $v);
			$h2_h6occ = substr_count($h2_h6Words, $v);
			$aocc = substr_count($linkTextWords, $v);
			$tocc = 0;
			$ialtocc = substr_count($imgAltTextWords, $v);
			$pocc = substr_count($pWords, $v);
		
			$pts = ($vars['metaK_h1Factor'] * $h1occ) + ($vars['metaK_stFactor'] * $stocc) + ($vars['metaK_domFactor'] * $docc) + ($vars['metaK_h2h6Factor'] * $h2_h6occ) + ($vars['metaK_aFactor'] * $aocc) + ($vars['metaK_ialtFactor'] * $ialtocc) + ($vars['metaK_tFactor'] * $tocc) + ($vars['metaK_pFactor'] * $pocc);		
			$keywordsArr[$v] = $pts;
			$allKeyWords = $allKeyWords." ".$v;
			
		}
		
		if($doubleWord != '')
		{
			$pts = $vars['metaKWDefault'];
			if(!array_key_exists($doubleWord, $keywordsArr))
			{
				
				$h1occ = 0;
				$docc = 0;
				$stocc = 0;
				$h2_h6occ = 0;
				$aocc = 0;
				$ialtocc = 0;
				$tocc = 0;
				$pocc = 0;

				$h1occ = substr_count($h1Words, $doubleWord);
				//$focc = substr_count($first100Words, $doubleWord);
				$docc = substr_count($domainWords, $doubleWord);
				$stocc = substr_count($strongWords, $doubleWord);
				$h2_h6occ = substr_count($h2_h6Words, $doubleWord);
				$ialtocc = substr_count($imgAltTextWords, $doubleWord);
				$aocc = substr_count($linkTextWords, $doubleWord);
				$tocc = 0;
				$pocc = substr_count($pWords, $doubleWord);

				$pts = ($vars['metaKW_h1Factor'] * $h1occ) + ($vars['metaKW_domFactor'] * $docc) + ($vars['metaKW_stFactor'] * $stocc) + ($vars['metaKW_h2h6Factor'] * $h2_h6occ) + ($vars['metaKW_aFactor'] * $aocc) + ($vars['metaKW_ialtFactor'] * $ialtocc) + ($vars['metaKW_tFactor'] * $tocc) + ($vars['metaKW_pFactor'] * $pocc);
	
				if($pts > 0)
				{
					$pts *= 4;
					$keywordsArr[$doubleWord] = $pts;
					$allKeyWords = $allKeyWords." ".$doubleWord;
				}
			}
			else
			{
				$pts *= 4;
				$pts += $keywordsArr[$doubleWord];
				$keywordsArr[$doubleWord] = $pts;
			}
		}
		$lastWord = $v;
	}
  }
  else if($mode == "h1" || $mode == "h2_h6")
  {
	if($mode == "h1")
		$def = $vars['h1Default'];
	else
		$def = $vars['h2_h6Default'];
		
	$lastWord = "";
	$titleWords = explode(" ", $string);
	foreach($titleWords as $k=>$v)
	{
		$skip = 0;
		$v = trim($v);
		$doubleWord = "";
		if(trim($lastWord) != '')
			$doubleWord = $lastWord." ".$v;
		
		$pts = $def;
		
		if(preg_match("/{$v}/",$allKeyWords) == 1)
			$skip = 1;
		if($skip != 1)
		{
			$docc = 0;
			$stocc = 0;
			$h2_h6occ = 0;
			$aocc = 0;
			$ialtocc = 0;
			$pocc = 0;

			$stocc = substr_count($strongWords, $v);
			$docc = substr_count($domainWords, $v);
			if($mode == "h1")
				$h2_h6occ = substr_count($h2_h6Words, $v);
			else
				$h2_h6occ = 0;
				
			$aocc = substr_count($linkTextWords, $v);
			$ialtocc = substr_count($imgAltTextWords, $v);
			$pocc = substr_count($pWords, $v);

			$pts = $def * ($stocc + $docc + $h2_h6occ + $aocc + $ialtocc + $pocc);
			$keywordsArr[$v] = $pts;
			$allKeyWords = $allKeyWords." ".$v;
		}
		
		if($doubleWord != '')
		{
			$pts = $def;
			if(!array_key_exists($doubleWord, $keywordsArr))
			{
				$h1occ = 0;
				$docc = 0;
				$stocc = 0;
				$h2_h6occ = 0;
				$aocc = 0;
				$ialtocc = 0;
				$pocc = 0;

				if($mode == "h1")
					$h1occ = substr_count($h1Words, $doubleWord);
				else
					$h1occ = 0;
				//$focc = substr_count($first100Words, $doubleWord);
				$docc = substr_count($domainWords, $doubleWord);
				$stocc = substr_count($strongWords, $doubleWord);
				if($mode == "h1")
					$h2_h6occ = substr_count($h2_h6Words, $doubleWord);
				else
					$h2_h6occ = 0;
				$ialtocc = substr_count($imgAltTextWords, $doubleWord);
				//$aocc = substr_count($linkTextWords, $doubleWord);
				$pocc = substr_count($pWords, $doubleWord);

				$pts = $def * ($stocc + $docc + $h2_h6occ + $ialtocc + $pocc);
				if($h1occ > 1)
				{
					$h1occ -= 1;
					$pts += ($def * $h1occ);
				}
				if($pts > $def)
				{
					$pts *= 4;
					$keywordsArr[$doubleWord] = $pts;
					$allKeyWords = $allKeyWords." ".$doubleWord;
				}
			}
			else
			{
				$pts *= 4;
				$pts += $keywordsArr[$doubleWord];
				$keywordsArr[$doubleWord];
			}
		}
		$lastWord = $v;
	}
  }
  else if($mode == "text")
  { 
	$lastWord = "";
	$titleWords = explode(" ", $string);
	foreach($titleWords as $k=>$v)
	{
		$skip = 0;
		$v = trim($v);
		$doubleWord = "";
		if(trim($lastWord) != '')
			$doubleWord = $lastWord." ".$v;
		
		$pts = 0;
		
		//$allKeyWords = implode_key(" ",$keywordsArr);
		if(preg_match("/{$v}/",$allKeyWords) == 1)
			$skip = 1;
		if($skip != 1)
		{
			$pocc = 0;
			$aocc = 0;
			$iocc = 0;
			$pocc = substr_count($pWords, $v);
			$aocc = substr_count($linkTextWords, $v);
			$iocc = substr_count($imgAltTextWords, $v);

			$pocc = $pocc + $aocc + $iocc - 1;

			if($pocc > 1)
			{
				$pts = ($vars['textDefault'] * $pocc);		
				$keywordsArr[$v] = $pts;
				$allKeyWords = $allKeyWords." ".$v;
				//echo "{$v} => {$pts}\n";
			}
		}
		
		if($doubleWord != '')
		{
			$pts = 0;
			if(!array_key_exists($doubleWord, $keywordsArr))
			{
				$pocc = 0;
				$aocc = 0;
				$iocc = 0;
				$pocc = substr_count($pWords, $doubleWord);	
				$aocc = substr_count($linkTextWords, $v);
				$iocc = substr_count($imgAltTextWords, $v);
				$pocc = $pocc + $aocc + $iocc - 1;

				if($pocc > 1)
				{	
					$pts = ($vars['textDefault'] * $pocc)*2;
					$keywordsArr[$doubleWord] = $pts;
					$allKeyWords = $allKeyWords." ".$doubleWord;
					//echo "{$doubleWord} => {$pts}\n";
				}
			}
			else
			{
				$pts = ($vars['textDefault']);
				$pts += $keywordsArr[$doubleWord];
				$keywordsArr[$doubleWord] = $pts;
			}
		}
		$lastWord = $v;
	}
  }
}

function getWordsLimit($html, $start, $limit)
{
	$ret = substr($html, $start, $limit);
	return $ret;
}

function getTitle($html)
{
	$ret = $html->find('title');
	$title = implode(" ", $ret);
	$title = ignoreWords($title);
	$title = ignoreChars($title);
	$title = removeMultiSpaces($title);
	return trim($title);
}

function getMetaInfo($html)
{
	$retArr = array();
	$pattern = '/<[\s]*meta[\s]*name="keywords"[\s]*(content|CONTENT)="?([^>"]*)"?[\s]*[\/]?[\s]*>/si';
	preg_match($pattern, $html, $matches);
	if(is_array($matches) && trim($matches[2])!= '')
	{
		$retArr['keywords'] = $matches[2];
	}
	else
	{
		$pattern = '/<[\s]*meta[\s]*http-equiv="keywords"[\s]*(content|CONTENT)="?([^>"]*)"?[\s]*[\/]?[\s]*>/si';
		preg_match($pattern, $html, $matches);
		if(is_array($matches) && trim($matches[2])!= '')
			$retArr['keywords'] = $matches[2];
	}
	//echo $retArr['keywords'];
	return $retArr;
}

function getMetaDescInfo($html)
{
	$retArr = array();
	$pattern = '/<[\s]*meta[\s]*name="description"[\s]*(content|CONTENT)="?([^>"]*)"?[\s]*[\/]?[\s]*>/si';
	preg_match($pattern, $html, $matches);
	if(is_array($matches) && trim($matches[2])!= '')
		$retArr['description'] = $matches[2];
	return $retArr;
}

function removeMultiSpaces($text)
{
	$text = preg_replace('/\s+/', ' ', $text);
	return $text;
}

function ignoreWords($text)
{
	global $commonWords;
	$text = strtolower($text);
	//$ignoreWords = array("your ","from ","only ","with ","more ","best ","product ","over ","about ","price ","where ","also ","edit ","have ","give ","take ","further ");
	$text = preg_replace("!\\b\\w{1,2}\\b!", "", $text); 
 
	$text = preg_replace('/\b('.implode('|',$commonWords).')\b/','',$text);
	//$text = preg_replace('/\b('.implode('|',$ignoreWords).')\b/','',$text);
	/*foreach($commonWords as $v)
	{
		$text = str_replace("{$v} ", "", $text);
	}*/
	return $text;
}

function ignoreChars($text)
{
	$text = preg_replace("/[^A-Za-z0-9 ' ]/", " ", $text);
	return $text;
}

function getH1Tags($html)
{
	$ret = $html->find('h1');
	$retString = implode(" ", $ret);
	$retString = stripText($retString);
	$retString = ignoreWords($retString);
	$retString = ignoreChars($retString);
	$retString = removeMultiSpaces($retString);
	return $retString;
}

function getH2_H6Tags($html)
{
	$retString = "";
	$h2_h6Tags = array("h2","h3","h4","h5","h6");
	if(!method_exists($html,"find"))
		return "";
	foreach($h2_h6Tags as $v)
	{
		$ret = $html->find($v);
		$retString .= implode(" ", $ret);
	}
	//$retString = stripText($retString);
	$retString = ignoreWords($retString);
	$retString = ignoreChars($retString);
	$retString = removeMultiSpaces($retString);
	return $retString;
}

function getStrongTags($html)
{
	$retString = "";
	$strongTags = array("b","strong","li");
	foreach($strongTags as $v)
	{
		$ret = $html->find($v);
		$retString .= implode(" ", $ret);
	}
	
	$retString = stripText($retString);
	$retString = ignoreWords($retString);
	$retString = ignoreChars($retString);
	$retString = removeMultiSpaces($retString);
	return $retString;
}

function getLinksText($html)
{
	$linkText = array();
	preg_match_all("/<a[^>]*?>(.*)<\/a>/i", $html, $matches);
	$retString = implode(" ", $matches[1]);
	$retString = stripText($retString);
	$retString = ignoreWords($retString);
	$retString = ignoreChars($retString);
	$retString = removeMultiSpaces($retString);
	//return $matches[0];
	return $retString;
}

function getPWords($html)
{
	$ret = $html->find('p');
	$retString = implode(" ", $ret);
	$retString = stripText($retString);
	$retString = ignoreWords($retString);
	$retString = ignoreChars($retString);
	$retString = removeMultiSpaces($retString);
	return $retString;
}

function getImgAltText($html)
{
	$pattern = '/<img[^>]+alt[\\s=\'"]+([^"\'>]+)/is';
	preg_match_all($pattern, $html, $matches);
	$retString = "";
	$retString = implode(" ",$matches[1]);

	$retString = stripText($retString);
	$retString = ignoreWords($retString);
	$retString = ignoreChars($retString);
	
	$retString = preg_replace('/(width|src|class|title)/i',"",$retString);
	$retString = removeMultiSpaces($retString);
	return $retString;
}

function getPageText($html, $htmlDOM, $webhost)
{
	global $textMinLimit;
	$retString = "";
	if(!strpos($webhost, "eutimes") && !strpos($webhost, "huffingtonpost"))
	{
		$pattern = '/(<!-- google_ad_section_start -->)(.*?)(<!-- google_ad_section_end -->)/is';
		preg_match_all($pattern, $html, $matches);
		$retString = implode(" ", $matches[2]);
	}
	//$pageTextHTML = $retString;
	$retString = stripSpecialTags($retString, 0);
	$retString = stripText($retString);

	if(trim($retString) == '')
	{
		$textTags = array("p","span","div");
		
		$html2 = stripSpecialTags($html);
		$htmlDOM = str_get_html($html2);
		$sMatches = array();
		$pMatches = array();
		$dMatches = array();

		if($htmlDOM && method_exists($htmlDOM,"find"))
		{	
			if($_GET['debug'] == "ms123")
				echo "DOM<br>";
			$pMatches = $htmlDOM->find('p');
			if(!strpos($webhost, "eutimes"))
				$pMatches = $htmlDOM->find('p');
			else
			{
				preg_match_all("/<p[^>]*?>(.*)<\/p>/i", $html, $matches);
				$pMatches = $matches[1];
			}
		}
		else
		{
			if($_GET['debug'] == "ms123")
				echo "No DOM<br>";
			preg_match_all("/<p[^>]*?>(.*)<\/p>/i", $html, $matches);
			$pMatches = $matches[1];
		}
		
		if($htmlDOM && method_exists($htmlDOM,"find"))
		{
			foreach($htmlDOM->find('span') as $e)
			{
				//echo $e->outertext."<br>";
				$v = stripSpecialTags($e->outertext);
				$v = stripText($v);
				if(strlen($v) > $textMinLimit)
					$retString .= $v;
			}
		}
		else
		{
			preg_match_all("/<span[^>]*?>(.*)<\/span>/i", $html, $matches);
			$spanMatches = $matches[1];
			foreach($spanMatches as $v)
			{
				$t = $v;
				
				$v = stripSpecialTags($t);
				$v = stripText($v);
				if(strlen($v) > 0)
					$retString .= $v;
			} 
		}
		
		foreach($pMatches as $v)
		{
			//echo $v."<br>";
			//$fp = strpos($v, "<p");
			//$lp = strrpos($v, "</p>");

			$t = $v;
			//$t = substr($v, 0, $fp);
			//$t .= substr($v, $lp, (strlen($v) - $lp));

			//echo $v."<br>";
			$v = stripSpecialTags($t);
			$v = stripText($v);
			if(strlen($v) > 0)
				$retString .= $v;
		} 
		
		if($htmlDOM && method_exists($htmlDOM,"find"))
		{
			foreach($htmlDOM->find('div') as $e)
			{
				//echo $e->innertext."<br>";
				$fdiv = strpos($e->innertext, "<div");
				$ldiv = strrpos($e->innertext, "</div>");

				$v = "";
				$v = substr($e->innertext, 0, $fdiv);
				$v .= substr($e->innertext, $ldiv, (strlen($e->innertext) - $ldiv));

				//$v = $e->innertext;
				$v = stripSpecialTags($v);
				$v = stripText($v);
				if(strlen($v) > $textMinLimit)
					$retString .= $v;
			}
		}
		else
		{
			preg_match_all("/<div[^>]*?>(.*)<\/div>/i", $html, $matches);
			$divMatches = $matches[1];
			foreach($divMatches as $v)
			{
				$t = $v;
				
				$v = stripSpecialTags($t);
				$v = stripText($v);
				if(strlen($v) > 0)
					$retString .= $v;
			} 
		}

		$pageTextHTML = $retString;
		/*$retString = ignoreWords($retString);
		$retString = ignoreChars($retString);
		$retString = removeMultiSpaces($retString);*/
	}

	if(strlen(trim($retString)) < 50)
	{
		$retString = stripText($html);
		//return $html;
	}
	return $retString;

}

function stripSpecialTags($html, $stripDiv = 1)
{
	$search = array("'<script[^>]*?>.*?</script>'si",
					"'<style[^>]*?>.*?</style>'si",
					"'<a[^>]*?>.*?</a>'si",
					"'<img[^>]+>'si",
					"'<title[^>]*?>.*?</title>'si",
					"'<select[^>]*?>.*?</select>'si",
					"'<strong[^>]*?>.*?</strong>'si",
					"'<b[^>]*?>.*?</b>'si",
					"'<i[^>]*?>.*?</i>'si",
					);
	if($stripDiv == 1)
	{
		
		$search[] = "'<div[^>]*?>.*?</div>'si";
	}
	$replace = array(	"",
						"",
						"",
						"",
						"",
						"",
						"",
						"",
						"",
					);
	if($stripDiv == 1)
	{
		
		$replace[] = "";
	}
	$html = preg_replace($search,$replace,$html);
		return $html;

}

function stripText($text)
{
	$search = array("'<script[^>]*?>.*?</script>'si",
					"'<style[^>]*?>.*?</style>'si",// strip out javascript
						"'<[\/\!]*?[^<>]*?>'si",			// strip out html tags
						"'([\r\n])[\s]+'",					// strip out white space
						"'&(quot|#34|#034|#x22);'i",		// replace html entities
						"'&(amp|#38|#038|#x26);'i",			// added hexadecimal values
						"'&(lt|#60|#060|#x3c);'i",
						"'&(gt|#62|#062|#x3e);'i",
						"'&(nbsp|#160|#xa0);'i",
						"'&(iexcl|#161);'i",
						"'&(cent|#162);'i",
						"'&(pound|#163);'i",
						"'&(copy|#169);'i",
						"'&(reg|#174);'i",
						"'&(deg|#176);'i",
						"'&(#39|#039|#x27);'",
						"'&(euro|#8364);'i",				// europe
						"'&a(uml|UML);'",					// german
						"'&o(uml|UML);'",
						"'&u(uml|UML);'",
						"'&A(uml|UML);'",
						"'&O(uml|UML);'",
						"'&U(uml|UML);'",
						"'&szlig;'i",
						"'&raquo;'i",
						"'&ldquo;'i",
						"'&rdquo;'i",
						"'&lsquo;'i",
						"'&rsquo;'i",
						"'&middot;'i",
						"'&ndash;'i",
						"'&mdash;'i",
						"'&iquest;'i",
						"'&laquo;'i",
						"'&divide;'i",
						"'&micro;'i",
						"'&para;'i",
						"'&plusmn;'i",
						"'&sect;'i",
						"'&trade;'i",
						"'&yen;'i",
						"'&aacute;'i",
						"'&agrave;'i",
						"'&acirc;'i",
						"'&aring;'i",
						"'&atilde;'i",
						"'&auml;'i",
						"'&aelig;'i",
						"'&ccedil;'i",
						"'&eacute;'i",
						"'&egrave;'i",
						"'&ecirc;'i",
						"'&euml;'i",
						"'&iacute;'i",
						"'&igrave;'i",
						"'&icirc;'i",
						"'&iuml;'i",
						"'&ntilde;'i",
						"'&oacute;'i",
						"'&ograve;'i",
						"'&ocirc;'i",
						"'&oslash;'i",
						"'&otilde;'i",
						"'&ouml;'i",
						"'&szlig;'i",
						"'&uacute;'i",
						"'&ugrave;'i",
						"'&ucirc;'i",
						"'&uuml;'i",
						"'&yuml;'i",
						"'&bull;'i",
						);
		$replace = array(	"",
							"",
							"",
							"\\1",
							"\"",
							"&",
							"<",
							">",
							" ",
							chr(161),
							chr(162),
							chr(163),
							chr(169),
							chr(174),
							chr(176),
							chr(39),
							chr(128),
							"�",
							"�",
							"�",
							"�",
							"�",
							"�",
							"�",
							"",
							"",
							"",
							"'",
							"'",
							"",
							"",
							"",
							"",
							"",
							"",
							"",
							"",
							"",
							"",
							"",
							"",
							"",
							"",
							"",
							"",
							"",
							"",
							"",
							"",
							"",
							"",
							"",
							"",
							"",
							"",
							"",
							"",
							"",
							"",
							"",
							"",
							"",
							"",
							"",
							"",
							"",
							"",
							"",
							"",
							"",
							"",
						);
					
		$text = preg_replace($search,$replace,$text);
		return $text;
}

function addHandle(&$curlHandle,$url)
{
	$cURL = curl_init();
	curl_setopt($cURL, CURLOPT_URL, $url);
	curl_setopt($cURL, CURLOPT_HEADER, 0);
	curl_setopt($cURL, CURLOPT_RETURNTRANSFER, 1);
	
	//curl_setopt($cURL, CURLOPT_CONNECTTIMEOUT, 2);
    curl_setopt($cURL, CURLOPT_AUTOREFERER, true);
    curl_setopt($cURL, CURLOPT_FAILONERROR, true);
    curl_setopt($cURL, CURLOPT_FOLLOWLOCATION, true);
    curl_setopt($cURL, CURLOPT_RETURNTRANSFER, true);
    curl_setopt($cURL, CURLOPT_SSL_VERIFYHOST, false);
    curl_setopt($cURL, CURLOPT_SSL_VERIFYPEER, false);
	curl_multi_add_handle($curlHandle,$cURL);
	return $cURL;
}	

function ExecHandle(&$curlHandle)
{
	$flag=null;
	do {
//fetch pages in parallel
		curl_multi_exec($curlHandle,$flag);
	} while ($flag > 0);
}

function microtime_float()
{
    list($usec, $sec) = explode(" ", microtime());
    return ((float)$usec + (float)$sec);
}
?>