auditor->find( array('p_id'=>P_ID, 'a_id' => $ca_id), array('_id'=>0, 'pos'=>1, 'url'=>1) ); foreach ($allUrls as $Ukey => $Uvalue) { $allUrlsArr[$Uvalue['pos']] = $Uvalue['url']; } if (!file_exists(DUPLICATE_DIR)) { if (!mkdir(DUPLICATE_DIR) || !chmod(DUPLICATE_DIR, 0777)) { log_error(P_ID, "file", error_get_last(), $mongoDB); } } $ts1 = $mongoDB->auditor->findOne( array('p_id'=>P_ID, 'a_id' => $ca_id, 'pos' => $start), array('_id'=>0, 'url'=>1, 'pos'=>1, 'dw'=>1) ); $firstSelect = array(); if(empty($_GET['end'])) { $i=0; foreach ($ts1['dw'] as $dpvalue) { if($i == 0) $end = $dpvalue['pos']; $urlfirsts = urldecode(urldecode(urldecode($allUrlsArr[$dpvalue['pos']]))); $firstSelect[$dpvalue['pos']] = $urlfirsts; // $firstSelect.= "".$urlfirsts.""; $i++; } } $second_url = $mongoDB->auditor->findOne( array('p_id'=>P_ID, 'a_id' => $ca_id, 'pos' => $end), array('_id'=>0, 'url'=>1, 'pos'=>1) ); $content1 = getPageData(urldecode($ts1['url'])); $content2 = getPageData(urldecode($second_url['url'])); if(empty($_GET['end'])) { if ($content1) { $dom=new DomDocument(); libxml_use_internal_errors(true); $dom->loadHTML($content1); if(!empty($dom->textContent)) { $links11 = $dom->getElementsByTagName('link'); if ($links11->length > 0) { foreach ($links11 as $link1) { $href1=$link1->attributes->getNamedItem("href")->value; if(!empty($href1)) { $u1 = processUrl($href1); $link1->attributes->getNamedItem("href")->value=$u1; // if (strpos(URL_TO_CRAWL, $href1)<=0) { // if(!preg_match('/((^http)|(\.(com|net|in)\/))/', $href1)) // { // if(preg_match('/^[\/]{2,}/', $href1)) // continue; // if(preg_match('/^[\/]/', $href1)) // $link1->attributes->getNamedItem("href")->value="https://".URL_TO_CRAWL.$href1; // else // $link1->attributes->getNamedItem("href")->value="https://".URL_TO_CRAWL."/".$href1; // } // } } } } $script11 = $dom->getElementsByTagName('script'); if ($script11->length > 0) { foreach ($script11 as $script1) { if(!empty($script1)) { $src1=$script1->attributes->getNamedItem("src")->value; $type1=$script1->attributes->getNamedItem("type")->value; if(empty($type1)) { $script1->attributes->getNamedItem("type")->value = "text/javascript"; } if(!empty($src1)) { $u2 = processUrl($src1); $script1->attributes->getNamedItem("src")->value=$u2; } } } } $images11 = $dom->getElementsByTagName('img'); if ($images11->length > 0) { foreach ($images11 as $image1) { if(!empty($image1)) { $src2=$image1->attributes->getNamedItem("src")->value; if(!empty($src2)) { $u3 = processUrl($src2); $image1->attributes->getNamedItem("src")->value=$u3; } } } } $jq_text = "var socket = new easyXDM.Socket({ onMessage: function(message, origin){ alert(message + origin); socket.postMessage('Indeed it does!'); } });"; $element = $dom->createElement('script', $jq_text); $domAttribute = $dom->createAttribute('type'); $domAttribute->value = 'text/javascript'; $element->appendChild($domAttribute); $head = $dom->getElementsByTagName('head')->item(0); $head->appendChild($element); $content1 = $dom->saveHTML(); } } $content1 = ( isset($content1) ) ? $content1 : ''; $overwrite = fopen(DUPLICATE_DIR."first.html", "w"); fwrite($overwrite, $content1); fclose($overwrite); $Overwrittenfile1 = URL_TO_CRAWL.".duplicate/"."first.html"; } // file name to store modified dom $matchFname = "second.html"; if ($content2) { $dom2=new DomDocument(); $dom2->loadHTML($content2); $links2 = $dom2->getElementsByTagName('link'); if ($links2->length > 0) { foreach ($links2 as $link2) { if(!empty($link2)) { $href2=$link2->attributes->getNamedItem("href")->value; if(!empty($href2)) { $u4 = processUrl($href2); $link2->attributes->getNamedItem("href")->value=$u4; } } } } $scripts2 = $dom2->getElementsByTagName('script'); if ($scripts2->length > 0) { foreach ($scripts2 as $script2) { if(!empty($script2)) { $src2=$script2->attributes->getNamedItem("src")->value; if(!empty($src2)) { $u5 = processUrl($src2); $script2->attributes->getNamedItem("src")->value=$u5; } } } } $imgs2 = $dom2->getElementsByTagName('img'); if ($imgs2->length > 0) { foreach ($imgs2 as $img2) { if(!empty($img2)) { $src2=$img2->attributes->getNamedItem("src")->value; if(!empty($src2)) { $u6 = processUrl($src2); $img2->attributes->getNamedItem("src")->value=$u6; } } } } $jq_text = "var socket = new easyXDM.Socket({ onMessage: function(message, origin){ alert(message + origin); socket.postMessage('Indeed it does!'); } });"; $element = $dom2->createElement('script', $jq_text); $domAttribute = $dom2->createAttribute('type'); $domAttribute->value = 'text/javascript'; $element->appendChild($domAttribute); $head = $dom2->getElementsByTagName('head')->item(0); $head->appendChild($element); $content2 = $dom2->saveHTML(); $diff = new HtmlDiff( $content1, $content2 ); $diff->build(); $matchHtml = $diff->getDifference(); } $matchHtml = ( isset($matchHtml) ) ? $matchHtml : ''; $wm = fopen(DUPLICATE_DIR.$matchFname, "w"); fwrite($wm, $matchHtml); fclose($wm); $name_file2 = "https://sauditor.rankwatch.com/Distributed/Auditor/Duplicates/".URL_TO_CRAWL.".duplicate/".$matchFname; if(empty($_GET['end'])) $name_file1 = "https://sauditor.rankwatch.com/Distributed/Auditor/Duplicates/".$Overwrittenfile1; else $name_file1 = ""; $result = array( 'file1' => $name_file1, 'file2' => $name_file2, 'select' => $firstSelect, 'url1' => rawurlencode($ts1['url']), 'url2' => rawurlencode($second_url['url']), 'response_code' => 100 ); echo json_encode($result); exit; function processUrl($url) { $uparsed1 = ""; $uparse1 = parse_url($url); if(!empty($uparse1['path'])) { if(!empty($uparse1['host'])) { $d_host = $uparse1['host']; } else { $d_host = URL_TO_CRAWL; } if(!empty($uparse1['query'])) { $d_qry = "?".$uparse1['query']; } else { $d_qry = ""; } if(!empty($uparse1['fragment'])) { $d_frag = "#".$uparse1['fragment']; } else { $d_frag = ""; } $uparsed1 = "http://".$d_host.$uparse1['path'].$d_qry.$d_frag; } return $uparsed1; } function getPageData($url){ $ch = curl_init(); $user_agent='Mozilla/5.0 (Windows NT 6.1; rv:8.0) Gecko/20100101 Firefox/8.0'; curl_setopt ($ch, CURLOPT_URL, $url); curl_setopt ($ch, CURLOPT_USERAGENT, $user_agent); curl_setopt ($ch, CURLOPT_HEADER, 0); curl_setopt ($ch, CURLOPT_FOLLOWLOCATION, 1); curl_setopt ($ch, CURLOPT_RETURNTRANSFER, 1); curl_setopt ($ch,CURLOPT_CONNECTTIMEOUT,30); curl_setopt ($ch,CURLOPT_TIMEOUT,30); curl_setopt ($ch,CURLOPT_MAXREDIRS,1); $content = curl_exec( $ch ); $err = curl_errno( $ch ); if ($err > 0 ) return false; $info = curl_getinfo($ch); if ($info['http_code'] != 200) return false; curl_close( $ch ); return $content; } ?>