Words //////////////////////////////////////////////////////////
$pWords = $pageText;
///////////////////Get Words from the Title ////////////////////////////////////////////////
unset($titleWords);
$titleWords = array();
if($html_ok)
{
$title = getTitle($htmlDOM);
if($title != '')
$titleWords = explode(" ", $title);
}
$lastWord = "";
foreach($titleWords as $k=>$v)
{
$skip = 0;
$v = trim($v);
$doubleWord = "";
if(trim($lastWord) != '')
$doubleWord = $lastWord." ".$v;
$pts = $vars['titleDefault'];
if($k == 0)
$pts = $vars['titleFirst'];
else if($k == 1)
$pts = $vars['titleSecond'];
else if($k == 2)
$pts = $vars['titleThird'];
else if($k == 3)
$pts = $vars['titleForth'];
//$allKeyWords = implode_key(" ",$keywordsArr);
if(preg_match("/{$v}/",$allKeyWords) == 1)
$skip = 1;
if($skip != 1)
{
$pts += getTitlePoints($v, 0);
}
else
$pts += $keywordsArr[$v];
$keywordsArr[$v] = $pts;
$allKeyWords = $allKeyWords." ".$v;
if($doubleWord != '')
{
$tPoints = 0;
if(!array_key_exists($doubleWord, $keywordsArr))
{
$tPoints = getTitlePoints($doubleWord, 1);
if($tPoints > 0)
{
$tPoints *= 3; //Multiply by 4 for importance of double words
$keywordsArr[$doubleWord] = $tPoints;
$allKeyWords = $allKeyWords." ".$doubleWord;
}
}
}
$lastWord = $v;
}
/////////////////////////////////////////////////////////////////////////////////////////////
///////////////////Get Words from Meta keyword tag according to frequency ////////////////////
///////////////////////////////////////////////////////////////////////////////////////////
if(trim($keyword_tag) != '')
{
//$keyword_tag = ignoreWords($keyword_tag);
//$keyword_meta = str_replace(" ", "", $keyword_meta);
$metaKeywords = explode(",", $keyword_tag);
//$metaKeywords = removeMultiSpaces($metaKeywords);
foreach($metaKeywords as $k=>$v)
{
$v = trim($v);
getWordsFromString($v, "meta");
}
}
if(trim($desc_tag) != '')
{
$desc_tag = ignoreChars($desc_tag);
$desc_tag = ignoreWords($desc_tag);
$desc_tag = removeMultiSpaces($desc_tag);
getWordsFromString($desc_tag, "meta");
}
/////////////////////////////////////////////////////////////////////////////////////////////
///////////////////Get Words from header tags according to frequency ///////////////////////
///////////////////////////////////////////////////////////////////////////////////////////
$h1Words = trim($h1Words);
getWordsFromString($h1Words, "h1");
getWordsFromString($h2_h6Words, "h2_h6");
getWordsFromString($pageText, "text");
/*
If $keywordsArr count is still less than 10
*/
if(count($keywordsArr) < 10)
{
if($imgAltTextWords != '')
{
$words = str_word_count($imgAltTextWords, 1);
$imgAltFreq= array_count_values($words);
arsort($imgAltFreq);
foreach($imgAltFreq as $ik=>$it)
{
if(count($keywordsArr) < 10)
$keywordsArr[$it] = 0.1;
}
}
}
if(count($keywordsArr) < 10)
{
if($linkTextWords != '')
{
$words = str_word_count($linkAltTextWords, 1);
$linkFreq= array_count_values($words);
arsort($linkFreq);
//echo count($linkFreq);
foreach($linkFreq as $ik=>$it)
{
if(count($keywordsArr) < 10)
$keywordsArr[$it] = 0.1;
}
}
}
//fwrite($fp, count($keywordsArr)."\n");
/****************************************************************************************************************
** Crawling process ends *
****************************************************************************************************************/
arsort($keywordsArr);
$i = 0;
$con = 0;
$inl = 0;
$insertKeys = "";
$inlineArr = array();
$occArr = array();
foreach($keywordsArr as $k=>$v)
{
if(!isset($occArr[$k]) || $occArr[$k] == '')
{
$kocc = preg_match_all("/\b{$k}\b/",$contextHTML, $wMatches);
$occArr[$k] = $kocc;
}
if(strpos($k, " "))
{
$rep = 0;
$wds = explode(" ", $k);
if(is_numeric(trim($wds[0])) || is_numeric(trim($wds[1])))
$k = "";
else if($wds[0] == $wds[1])
$k = "";
else if(preg_match('/[0-9]/',$wds[0]) == 1)
$k = $wds[1];
else if(preg_match('/[0-9]/',$wds[1]) == 1)
$k = $wds[0];
else
{
//unset($keywordsArr[$wds[0]]);
//$insertKeys = str_replace(",{$wds[0]},",",",$insertKeys,$rep);
//$i -= $rep;
//$con -= $rep;
}
}
$plural = $k."s";
$nspace = str_replace(" ","",trim($k));
$hostok = 1;
if(preg_match("/{$nspace}/",$webhost) == 1)
{
$hostok = 0;
}
if(($i < $vars['keyWordsInsertLimit'] || $inl < $vars['minInlineLimit']) && strlen($k) > 2 && !is_numeric($k) && $hostok == 1)
{
$k = trim($k);
if($con < $vars['contextLimit'])
{
$skip = 0;
if(preg_match("/\b{$k}\b/",$insertKeys) == 1)
$skip = 1;
else if(preg_match('/[0-9]/',$k) == 1 && $k != '5zigen')
$skip = 1;
if($skip == 0)
{
// Multiple words ///
$nWordCon = "";
foreach($keywordsArr as $ck=>$cv)
{
if(trim($ck) != '' && trim($nWordCon) == '' && $ck != $k)
{
//$test .= $r.",";
if(!isset($occArr[$ck]) || $occArr[$ck] == '')
{
$ckocc = preg_match_all("/\b{$ck}\b/",$contextHTML, $wMatches);
$occArr[$ck] = $ckocc;
}
else
$ckocc = $occArr[$ck];
$n = 0;
$nWord = $k." ".$ck;
$nWordCh = preg_replace("/([,.?!])/"," \\1",$nWord);
$nWordSp = explode(" ",$nWordCh);
$nWordSp = array_unique($nWordSp);
$nWord = implode(" ",$nWordSp);
$kckocc = substr_count($contextHTML, $nWord);
//echo "K: {$k}, CK: {$ck}, kocc: {$kocc}, ckocc: {$ckocc}, kckocc: {$kckocc}
";
if(preg_match("/\b{$nWord}\b/",$contextHTML) == 1 && substr_count($nWord, " ") > 0 && $kckocc >= ($kocc + $ckocc)/10 && str_word_count($nWord) <= 2)
{
$nWordCon = $nWord;
$insertKeys .= $nWordCon;
$insertKeys .= ",";
$con++;
//unset($keywordsArr[$ck]);
}
}
}
if(preg_match("/\b{$k}\b/",$contextHTML) == 1 && $nWordCon == '' && str_word_count($k) <= 2)
{
$insertKeys .= $k;
$insertKeys .= ",";
$con++;
}
}
}
if($cType == "i" && strpos($pageTextHTML, " ".$k." ") !== false && !array_key_exists($plural, $inlineArr) && substr($k, -3) != "ing" && str_word_count($k) <= 2)
{
$inlineKeys .= $k;
$inlineKeys .= ",";
$inlineArr[$k] = $v;
$inl++;
}
$i++;
}
}
//foreach($inlineBidArr as $ib=>$ipt)
//echo "{$ib} --> {$ipt}
";
/// Related Words ////////////////
$expire = 60*10; // 10 minutes
$cntCommas = substr_count($insertKeys, ",");
//echo $cntCommas;
for($cc=0; $cc < (3-$cntCommas); $cc++)
{
foreach($contextK as $cont)
if(!strstr($insertKeys, $cont))
{
$toAdd = $cont;
break;
}
$insertKeys .= $cont.",";
}
if($insertKeys == ",,,")
$insertKeys = "";
//echo "IN : {$insertKeys}
";
$contextualBids = "";
if($cType == "i")
{
//$inArr = explode(",", $inlineKeys, 2);
//$relatedInline = implode(",",$relatedInlineArr);
//$inlineKeys = $inArr[0].",".$relatedInline.",".$inArr[1];
//$inlineKeysArr = explode(",",$inlineKeys);
$inlineKeysArr = $inlineArr;
$newInline = array();
$inlineKeys = "";
$skipV = 0;
$test = "";
$singleWords = 0;
foreach($inlineKeysArr as $v=>$p1)
{
$skipV = 0;
if(trim($v) != '')
{
$no = 0;
foreach($inlineKeysArr as $r=>$p2)
{
if(trim($r) != '')
{
//$test .= $r.",";
$n = 0;
$nWord = $v." ".$r;
$nWordCh = preg_replace("/([,.?!])/"," \\1",$nWord);
$nWordSp = explode(" ",$nWordCh);
$nWordSp = array_unique($nWordSp);
$nWord = implode(" ",$nWordSp);
if(strpos($pageTextHTML, " ".$nWord." ") && substr_count($nWord, " ") > 0)
{
//$test .= $nWord."=> $v,$r\n";
//$n = str_word_count($nWord,0);
$singular = "";
$lChar = substr($nWord, -1);
if($lChar == "s")
$singular = substr($nWord, 0, -1);
if(($singular == '' || !array_key_exists($singular, $newInline)) && str_word_count($nWord) <= 2)
{
$newInline[$nWord] = ($p1 + $p2);
$skipV = 1;
}
//$inlineKeys .= $nWord;
//$inlineKeys .= ",";
}
}
}
if($skipV == 0)
{
$singular = "";
$lChar = substr(trim($v), -1);
if($lChar == "s")
$singular = substr($v, 0, -1);
if($singular == '' || !array_key_exists($singular, $newInline))
{
if(!strpos($v, " "))
{
if($singleWords < 10)
{
//$no = str_word_count($v,0);
$newInline[$v] = $p1;
$singleWords++;
}
}
else
{
//$no = str_word_count($v,0);
$newInline[$v] = $p1;
}
//$inlineKeys .= $v;
//$inlineKeys .= ",";
}
}
}
}
arsort($newInline);
$inlineKeys = implode(",", $advMatch_kwords);
if($inlineKeys != '')
$inlineKeys .= ",".implode_key(",",$newInline);
else
$inlineKeys = implode_key(",", $newInline);
//$inlineKeys = $pageTextHTML;
//echo $inlineKeys;
}
// Database Operations Start //
$in = 0;
$md5key = md5("crawler_".$webpage);
$n++;
}
function getAdvKeywords($pageTextHTML)
{
if($_GET['debug'] == "ms123")
echo "Entered getAdvKeywords
";
$adv_keywords = array();
$expire = "900";
$mkey = md5("crawler_advKeywords2");
if(!$adv_keywords = mCache::mGet()->get($mkey)){
if($_GET['debug'] == "ms123")
echo "No MC
";
$db = ConnectToCPV_List();
//@mysql_select_db ("Admin", $db);
if($_GET['debug'] == "ms123")
{
if($db)
echo "DB Conn
";
else
echo "No DB
";
}
$kSql = "SELECT ck.keyword from campaign_keywords ck left join campaign c on (ck.campaign_id = c.id) where c.status = 'A' order by ck.bid desc";
$kRes = mysql_query($kSql, $db);
while($kRow = mysql_fetch_assoc($kRes))
{
$adv_keywords[] = $kRow['keyword'];
}
if($_GET['debug'] == "ms123")
echo "Campaign_keywords ".count($adv_keywords)."
";
$bSql = "SELECT bk.keyword from ads_campaign_keywords_broadmatch bk left join campaign c on (bk.campaign_id = c.id) where c.status = 'A' order by bk.bid desc";
$bRes = mysql_query($bSql, $db);
while($bRow = mysql_fetch_assoc($bRes))
{
$adv_keywords[] = $bRow['keyword'];
}
if($_GET['debug'] == "ms123")
echo "ads_campaign_keywords_broadmatch ".count($adv_keywords)."
";
mCache::mGet()->set($mkey, $adv_keywords, 0, $expire);
}
if($_GET['debug'] == "ms123")
echo "Matching Kwords ".count($adv_keywords)."
";
$matched_keywords = array();
$adv_keywords_chunk = array_chunk($adv_keywords, 250000);
$adv_keywords = $adv_keywords_chunk[0];
//foreach($pageTextArr as $pageTextWord)
foreach($adv_keywords as $keyword)
{
$keyword = preg_replace("/[^A-Za-z0-9 '\s-_]/", "", $keyword);
if($keyword != '' && !in_array($keyword, $matched_keywords) && strpos($pageTextHTML, " ".$keyword." ") !== false)
{
$matched_keywords[] = $keyword;
if($_GET['debug'] == "ms123")
echo count($matched_keywords).",";
if(count($matched_keywords) == 15)
break;
}
/*if($pageTextWord != '' && !in_array($pageTextWord, $matched_keywords) && in_array($pageTextWord, $adv_keywords))
{
$matched_keywords[] = $pageTextWord;
if($_GET['debug'] == "ms123")
echo count($matched_keywords).",";
}*/
}
if($_GET['debug'] == "ms123")
echo "
Matching done
";
return $matched_keywords;
}
function sortByLength($a,$b){
if(str_word_count($a) == str_word_count($b)) return 0;
return (str_word_count($a) > str_word_count($b) ? -1 : 1);
}
function getTitlePoints($word, $isDouble=0)
{
global $vars;
global $h1Words, $domainWords, $strongWords, $h2_h6Words, $imgAltTextWords, $textOnly, $linkTextWords, $pWords;
$pts = 0;
$h1occ = 0;
$docc = 0;
$stocc = 0;
$h2_h6occ = 0;
$aocc = 0;
$ialtocc = 0;
$tocc = 0;
$h1occ = substr_count($h1Words, $word);
//$focc = substr_count($first100Words, $doubleWord);
$docc = substr_count($domainWords, $word);
$stocc = substr_count($strongWords, $word);
$h2_h6occ = substr_count($h2_h6Words, $word);
$ialtocc = substr_count($imgAltTextWords, $word);
$tocc = 0;
$aocc = substr_count($linkTextWords, $word);
$pocc = substr_count($pWords, $word);
//$tocc -= $pocc;
if($isDouble == 1)
{
$pts = ($vars['title_h1Factor'] * $h1occ) + ($vars['title_domFactor'] * $docc) + ($vars['title_stFactor'] * $stocc) + ($vars['title_h2h6Factor'] * $h2_h6occ) + ($vars['title_ialtFactor'] * $ialtocc) + ($vars['title_aFactor'] * $aocc) + ($vars['title_pFactor'] * $pocc);
}
else
{
$pts += ($vars['title_h1Factor'] * $h1occ) + ($vars['title_domFactor'] * $docc) + ($vars['title_stFactor'] * $stocc) + ($vars['title_h2h6Factor'] * $h2_h6occ) + ($vars['title_aFactor'] * $aocc) + ($vars['title_ialtFactor'] * $ialtocc) + ($vars['title_tFactor'] * $tocc) + ($vars['title_pFactor'] * $pocc);
}
return $pts;
}
function implode_key($glue = " ", $pieces = array())
{
$arrK = array_keys($pieces);
return implode($glue, $arrK);
}
function getWordsFromString($string, $mode)
{
global $keywordsArr, $allKeyWords;
global $h1Words, $strongWords, $domainWords, $h2_h6Words, $linkTextWords, $textOnly, $imgAltTextWords, $pWords;
global $vars;
//echo "{$h1Words}
{$strongWords}
{$domainWords}
{$h2_h6Words}";
if($mode == "meta")
{
$lastWord = "";
$titleWords = explode(" ", $string);
foreach($titleWords as $k=>$v)
{
$skip = 0;
$v = trim($v);
$doubleWord = "";
if(trim($lastWord) != '')
$doubleWord = $lastWord." ".$v;
$pts = 0;
//$allKeyWords = implode_key(" ",$keywordsArr);
if(preg_match("/{$v}/",$allKeyWords) == 1)
$skip = 1;
if($skip != 1)
{
$h1occ = 0;
$docc = 0;
$stocc = 0;
$h2_h6occ = 0;
$aocc = 0;
$ialtocc = 0;
$tocc = 0;
$pocc = 0;
$h1occ = substr_count($h1Words, $v);
//$focc = substr_count($first100Words, $v);
$stocc = substr_count($strongWords, $v);
$docc = substr_count($domainWords, $v);
$h2_h6occ = substr_count($h2_h6Words, $v);
$aocc = substr_count($linkTextWords, $v);
$tocc = 0;
$ialtocc = substr_count($imgAltTextWords, $v);
$pocc = substr_count($pWords, $v);
$pts = ($vars['metaK_h1Factor'] * $h1occ) + ($vars['metaK_stFactor'] * $stocc) + ($vars['metaK_domFactor'] * $docc) + ($vars['metaK_h2h6Factor'] * $h2_h6occ) + ($vars['metaK_aFactor'] * $aocc) + ($vars['metaK_ialtFactor'] * $ialtocc) + ($vars['metaK_tFactor'] * $tocc) + ($vars['metaK_pFactor'] * $pocc);
$keywordsArr[$v] = $pts;
$allKeyWords = $allKeyWords." ".$v;
}
if($doubleWord != '')
{
$pts = $vars['metaKWDefault'];
if(!array_key_exists($doubleWord, $keywordsArr))
{
$h1occ = 0;
$docc = 0;
$stocc = 0;
$h2_h6occ = 0;
$aocc = 0;
$ialtocc = 0;
$tocc = 0;
$pocc = 0;
$h1occ = substr_count($h1Words, $doubleWord);
//$focc = substr_count($first100Words, $doubleWord);
$docc = substr_count($domainWords, $doubleWord);
$stocc = substr_count($strongWords, $doubleWord);
$h2_h6occ = substr_count($h2_h6Words, $doubleWord);
$ialtocc = substr_count($imgAltTextWords, $doubleWord);
$aocc = substr_count($linkTextWords, $doubleWord);
$tocc = 0;
$pocc = 0;
$pts = ($vars['metaKW_h1Factor'] * $h1occ) + ($vars['metaKW_domFactor'] * $docc) + ($vars['metaKW_stFactor'] * $stocc) + ($vars['metaKW_h2h6Factor'] * $h2_h6occ) + ($vars['metaKW_aFactor'] * $aocc) + ($vars['metaKW_ialtFactor'] * $ialtocc) + ($vars['metaKW_tFactor'] * $tocc) + ($vars['metaKW_pFactor'] * $pocc);
if($pts > 0)
{
$pts *= 4;
$keywordsArr[$doubleWord] = $pts;
$allKeyWords = $allKeyWords." ".$doubleWord;
}
}
else
{
$pts *= 4;
$pts += $keywordsArr[$doubleWord];
$keywordsArr[$doubleWord] = $pts;
}
}
$lastWord = $v;
}
}
else if($mode == "h1" || $mode == "h2_h6")
{
if($mode == "h1")
$def = $vars['h1Default'];
else
$def = $vars['h2_h6Default'];
$lastWord = "";
$titleWords = explode(" ", $string);
foreach($titleWords as $k=>$v)
{
$skip = 0;
$v = trim($v);
$doubleWord = "";
if(trim($lastWord) != '')
$doubleWord = $lastWord." ".$v;
$pts = $def;
if(preg_match("/{$v}/",$allKeyWords) == 1)
$skip = 1;
if($skip != 1)
{
$docc = 0;
$stocc = 0;
$h2_h6occ = 0;
$aocc = 0;
$ialtocc = 0;
$pocc = 0;
$stocc = substr_count($strongWords, $v);
$docc = substr_count($domainWords, $v);
if($mode == "h1")
$h2_h6occ = substr_count($h2_h6Words, $v);
else
$h2_h6occ = 0;
$aocc = substr_count($linkTextWords, $v);
$ialtocc = substr_count($imgAltTextWords, $v);
$pocc = 0;
$pts = $def * ($stocc + $docc + $h2_h6occ + $aocc + $ialtocc + $pocc);
$keywordsArr[$v] = $pts;
$allKeyWords = $allKeyWords." ".$v;
}
if($doubleWord != '')
{
$pts = $def;
if(!array_key_exists($doubleWord, $keywordsArr))
{
$h1occ = 0;
$docc = 0;
$stocc = 0;
$h2_h6occ = 0;
$aocc = 0;
$ialtocc = 0;
$pocc = 0;
if($mode == "h1")
$h1occ = substr_count($h1Words, $doubleWord);
else
$h1occ = 0;
//$focc = substr_count($first100Words, $doubleWord);
$docc = substr_count($domainWords, $doubleWord);
$stocc = substr_count($strongWords, $doubleWord);
if($mode == "h1")
$h2_h6occ = substr_count($h2_h6Words, $doubleWord);
else
$h2_h6occ = 0;
$ialtocc = substr_count($imgAltTextWords, $doubleWord);
//$aocc = substr_count($linkTextWords, $doubleWord);
$pocc = 0;
$pts = $def * ($stocc + $docc + $h2_h6occ + $ialtocc + $pocc);
if($h1occ > 1)
{
$h1occ -= 1;
$pts += ($def * $h1occ);
}
if($pts > $def)
{
$pts *= 4;
$keywordsArr[$doubleWord] = $pts;
$allKeyWords = $allKeyWords." ".$doubleWord;
}
}
else
{
$pts *= 4;
$pts += $keywordsArr[$doubleWord];
$keywordsArr[$doubleWord];
}
}
$lastWord = $v;
}
}
else if($mode == "text")
{
$lastWord = "";
$titleWords = explode(" ", $string);
foreach($titleWords as $k=>$v)
{
$skip = 0;
$v = trim($v);
$doubleWord = "";
if(trim($lastWord) != '')
$doubleWord = $lastWord." ".$v;
$pts = 0;
//$allKeyWords = implode_key(" ",$keywordsArr);
if(preg_match("/{$v}/",$allKeyWords) == 1)
$skip = 1;
if($skip != 1)
{
$pocc = 0;
$aocc = 0;
$iocc = 0;
$pocc = substr_count($pWords, $v);
$aocc = substr_count($linkTextWords, $v);
$iocc = substr_count($imgAltTextWords, $v);
$pocc = $pocc + $aocc + $iocc - 1;
if($pocc > 1)
{
$pts = ($vars['textDefault'] * $pocc);
$keywordsArr[$v] = $pts;
$allKeyWords = $allKeyWords." ".$v;
//echo "{$v} => {$pts}\n";
}
}
if($doubleWord != '')
{
$pts = 0;
if(!array_key_exists($doubleWord, $keywordsArr))
{
$pocc = 0;
$aocc = 0;
$iocc = 0;
$pocc = substr_count($pWords, $doubleWord);
$aocc = substr_count($linkTextWords, $v);
$iocc = substr_count($imgAltTextWords, $v);
$pocc = $pocc + $aocc + $iocc - 1;
if($pocc > 1)
{
$pts = ($vars['textDefault'] * $pocc)*2;
$keywordsArr[$doubleWord] = $pts;
$allKeyWords = $allKeyWords." ".$doubleWord;
//echo "{$doubleWord} => {$pts}\n";
}
}
else
{
$pts = ($vars['textDefault']);
$pts += $keywordsArr[$doubleWord];
$keywordsArr[$doubleWord] = $pts;
}
}
$lastWord = $v;
}
}
}
function getWordsLimit($html, $start, $limit)
{
$ret = substr($html, $start, $limit);
return $ret;
}
function getTitle($html)
{
$ret = $html->find('title');
$title = implode(" ", $ret);
$title = ignoreWords($title);
$title = ignoreChars($title);
$title = removeMultiSpaces($title);
return trim($title);
}
function getMetaInfo($html)
{
$retArr = array();
$pattern = '/<[\s]*meta[\s]*name="keywords"[\s]*(content|CONTENT)="?([^>"]*)"?[\s]*[\/]?[\s]*>/si';
preg_match($pattern, $html, $matches);
if(is_array($matches) && trim($matches[2])!= '')
{
$retArr['keywords'] = $matches[2];
}
else
{
$pattern = '/<[\s]*meta[\s]*http-equiv="keywords"[\s]*(content|CONTENT)="?([^>"]*)"?[\s]*[\/]?[\s]*>/si';
preg_match($pattern, $html, $matches);
if(is_array($matches) && trim($matches[2])!= '')
$retArr['keywords'] = $matches[2];
}
//echo $retArr['keywords'];
return $retArr;
}
function getMetaDescInfo($html)
{
$retArr = array();
$pattern = '/<[\s]*meta[\s]*name="description"[\s]*(content|CONTENT)="?([^>"]*)"?[\s]*[\/]?[\s]*>/si';
preg_match($pattern, $html, $matches);
if(is_array($matches) && trim($matches[2])!= '')
$retArr['description'] = $matches[2];
return $retArr;
}
function removeMultiSpaces($text)
{
$text = preg_replace('/\s+/', ' ', $text);
return $text;
}
function ignoreWords($text)
{
global $commonWords;
$text = strtolower($text);
//$ignoreWords = array("your ","from ","only ","with ","more ","best ","product ","over ","about ","price ","where ","also ","edit ","have ","give ","take ","further ");
$text = preg_replace("!\\b\\w{1,2}\\b!", "", $text);
$text = preg_replace('/\b('.implode('|',$commonWords).')\b/','',$text);
//$text = preg_replace('/\b('.implode('|',$ignoreWords).')\b/','',$text);
/*foreach($commonWords as $v)
{
$text = str_replace("{$v} ", "", $text);
}*/
return $text;
}
function ignoreChars($text)
{
$text = preg_replace("/[^A-Za-z0-9 ' ]/", " ", $text);
return $text;
}
function getH1Tags($html)
{
$ret = $html->find('h1');
$retString = implode(" ", $ret);
$retString = stripText($retString);
$retString = ignoreWords($retString);
$retString = ignoreChars($retString);
$retString = removeMultiSpaces($retString);
return $retString;
}
function getH2_H6Tags($html)
{
$retString = "";
$h2_h6Tags = array("h2","h3","h4","h5","h6");
foreach($h2_h6Tags as $v)
{
$ret = $html->find($v);
$retString .= implode(" ", $ret);
}
$retString = stripText($retString);
$retString = ignoreWords($retString);
$retString = ignoreChars($retString);
$retString = removeMultiSpaces($retString);
return $retString;
}
function getStrongTags($html)
{
$retString = "";
$strongTags = array("b","strong","li");
foreach($strongTags as $v)
{
$ret = $html->find($v);
$retString .= implode(" ", $ret);
}
$retString = stripText($retString);
$retString = ignoreWords($retString);
$retString = ignoreChars($retString);
$retString = removeMultiSpaces($retString);
return $retString;
}
function getLinksText($html)
{
$linkText = array();
preg_match_all("/
";
$v = stripSpecialTags($e->outertext);
$v = stripText($v);
if(strlen($v) > $textMinLimit)
$retString .= $v;
}
foreach($pMatches as $v)
{
//echo $v."
";
//$fp = strpos($v, "
";
$v = stripSpecialTags($t);
$v = stripText($v);
if(strlen($v) > 0)
$retString .= $v;
}
foreach($htmlDOM->find('div') as $e)
{
//echo $e->innertext."
";
$fdiv = strpos($e->innertext, "