auditor->find( array('pi'=>P_ID, 'w' => $week), array('_id'=>0, 'pos'=>1, 'u'=>1) ); foreach ($allUrls as $Ukey => $Uvalue) { $allUrlsArr[$Uvalue['pos']] = $Uvalue['u']; } if (!file_exists(DUPLICATE_DIR)) { if (!mkdir(DUPLICATE_DIR) || !chmod(DUPLICATE_DIR, 0777)) { log_error(P_ID, "file", error_get_last(), $mongoDB); } } $ts1 = $mongoDB->auditor->findOne( array('pi'=>P_ID, 'w' => $week, 'pos' => $start), array('_id'=>0, 'u'=>1, 'pos'=>1, 'dw'=>1) ); $firstSelect = array(); if(empty($_GET['end'])) { $i=0; foreach ($ts1['dw'] as $dpvalue) { if($i == 0) $end = $dpvalue['pos']; $urlfirsts = urldecode(urldecode(urldecode($allUrlsArr[$dpvalue['pos']]))); $firstSelect[$dpvalue['pos']] = $urlfirsts; // $firstSelect.= "".$urlfirsts.""; $i++; } } $second_url = $mongoDB->auditor->findOne( array('pi'=>P_ID, 'w' => $week, 'pos' => $end), array('_id'=>0, 'u'=>1, 'pos'=>1) ); $content2 = file_get_contents(urldecode(urldecode(urldecode($second_url['u'])))); $content1 = file_get_contents(urldecode(urldecode(urldecode($ts1['u'])))); if(empty($_GET['end'])) { $dom=new DomDocument(); libxml_use_internal_errors(true); $dom->loadHTML($content1); if(!empty($dom->textContent)) { $links11 = $dom->getElementsByTagName('link'); foreach ($links11 as $link1) { $href1=$link1->attributes->getNamedItem("href")->value; if(!empty($href1)) { if (strpos(URL_TO_CRAWL, $href1)<=0) { if(!preg_match('/((^http)|(\.(com|net|in)\/))/', $href1)) { if(preg_match('/^[\/]{2,}/', $href1)) continue; if(preg_match('/^[\/]/', $href1)) $link1->attributes->getNamedItem("href")->value="https://".URL_TO_CRAWL.$href1; else $link1->attributes->getNamedItem("href")->value="https://".URL_TO_CRAWL."/".$href1; } } } } $script11 = $dom->getElementsByTagName('script'); foreach ($script11 as $script1) { if(!empty($script1)) { $src1=$script1->attributes->getNamedItem("src")->value; $type1=$script1->attributes->getNamedItem("type")->value; if(empty($type1)) { $script1->attributes->getNamedItem("type")->value = "text/javascript"; } if(!empty($src1)) { if (strpos(URL_TO_CRAWL, $src1)<=0) { if(!preg_match('/((^http)|(\.(com|net|in|co)\/))/', $src1)) { if(preg_match('/^[\/]{2,}/', $src1)) continue; if(preg_match('/^[\/]/', $src1)) $script1->attributes->getNamedItem("src")->value="http://".URL_TO_CRAWL.$src1; else $script1->attributes->getNamedItem("src")->value="http://".URL_TO_CRAWL."/".$src1; } } } } } $images11 = $dom->getElementsByTagName('img'); foreach ($images11 as $image1) { if(!empty($image1)) { $src2=$image1->attributes->getNamedItem("src")->value; if(!empty($src2)) { if (strpos(URL_TO_CRAWL, $src2)<=0) { if(!preg_match('/((^http)|(\.(com|net|in|co)\/))/', $src2)) { if(preg_match('/^[\/]{2,}/', $src2)) continue; if(preg_match('/^[\/]/', $src2)) $image1->attributes->getNamedItem("src")->value="http://".URL_TO_CRAWL.$src2; else $image1->attributes->getNamedItem("src")->value="http://".URL_TO_CRAWL."/".$src2; } } } } } $content1 = $dom->saveHTML(); $overwrite = fopen(DUPLICATE_DIR."first.html", "w"); fwrite($overwrite, $content1); fclose($overwrite); } $Overwrittenfile1 = URL_TO_CRAWL.".duplicate/"."first.html"; } // file name to store modified dom $matchFname = "second.html"; $dom2=new DomDocument(); $dom2->loadHTML($content2); $links2 = $dom2->getElementsByTagName('link'); foreach ($links2 as $link2) { if(!empty($link2)) { $href2=$link2->attributes->getNamedItem("href")->value; if(!empty($href2)) { if (strpos(URL_TO_CRAWL, $href2)<=0) { if(!preg_match('/((^http)|(\.(com|net|in|co)\/))/', $href2)) { if(preg_match('/^[\/]{2,}/', $href2)) continue; if(preg_match('/^[\/]/', $href2)) $link2->attributes->getNamedItem("href")->value="http://".URL_TO_CRAWL.$href2; else $link2->attributes->getNamedItem("href")->value="http://".URL_TO_CRAWL."/".$href2; } } } } } $scripts2 = $dom2->getElementsByTagName('script'); foreach ($scripts2 as $script2) { if(!empty($script2)) { $src2=$script2->attributes->getNamedItem("src")->value; if(!empty($src2)) { if (strpos(URL_TO_CRAWL, $src2)<=0) { if(!preg_match('/((^http)|(\.(com|net|in|co)\/))/', $src2)) { if(preg_match('/^[\/]{2,}/', $src2)) continue; if(preg_match('/^[\/]/', $src2)) $script2->attributes->getNamedItem("src")->value="http://".URL_TO_CRAWL.$src2; else $script2->attributes->getNamedItem("src")->value="http://".URL_TO_CRAWL."/".$src2; } } } } } $imgs2 = $dom2->getElementsByTagName('img'); foreach ($imgs2 as $img2) { if(!empty($img2)) { $src2=$img2->attributes->getNamedItem("src")->value; if(!empty($src2)) { if (strpos(URL_TO_CRAWL, $src2)<=0) { if(!preg_match('/((^http)|(\.(com|net|in)\/))/', $src2)) { if(preg_match('/^[\/]{2,}/', $src2)) continue; if(preg_match('/^[\/]/', $src2)) $img2->attributes->getNamedItem("src")->value="http://".URL_TO_CRAWL.$src2; else $img2->attributes->getNamedItem("src")->value="http://".URL_TO_CRAWL."/".$src2; } } } } } $content2 = $dom2->saveHTML(); $diff = new HtmlDiff( $content1, $content2 ); $diff->build(); $matchHtml = $diff->getDifference(); $wm = fopen(DUPLICATE_DIR.$matchFname, "w"); fwrite($wm, $matchHtml); fclose($wm); $name_file2 = "https://sauditor.rankwatch.com/Distributed/Auditor/Duplicates/".URL_TO_CRAWL.".duplicate/".$matchFname; if(empty($_GET['end'])) $name_file1 = "https://sauditor.rankwatch.com/Distributed/Auditor/Duplicates/".$Overwrittenfile1; else $name_file1 = ""; $result = array( 'file1' => $name_file1, 'file2' => $name_file2, 'select' => $firstSelect, 'url1' => rawurlencode($ts1['u']), 'url2' => rawurlencode($second_url['u']), 'response_code' => 100 ); echo json_encode($result); exit; ?>