mongoDB = $mongo->selectDB("distributed");
$this->pid = $p_id;
if($project !== "") {
$this->domain = $project;
}
if($url_pos !== "") {
$this->pos = (int)$url_pos;
}
if($week_no !== "") {
$this->week = (int)$week_no;
}
}
public function sanitizeCurlFailureCause($err)
{
$err_msg = "";
if(strstr($err, '1,')) {
$err_msg = "Invalid Transport Protocol";
}
elseif(strstr($err, '2,')) {
$err_msg = "Downloader Internal Error";
}
elseif(strstr($err, '3,')) {
$err_msg = "The URL was not properly formatted.";
}
elseif(strstr($err, '4,')) {
$err_msg = "Downloader Internal Error";
}
elseif(strstr($err, '5,')) {
$err_msg = "Unable resolve proxy.";
}
elseif(strstr($err, '6,')) {
$err_msg = "Unable to resolve Host.";
}
elseif(strstr($err, '7,')) {
$err_msg = "Unable to connect to host";
}
elseif(strstr($err, '8,')) {
$err_msg = "Bad response from server";
}
elseif(strstr('9,', $err)) {
$err_msg = "Access denied on server";
}
// ftp errors
// elseif(strstr('10,', $err)) {
// $err_msg = "Invalid Transport Protocol";
// }
// elseif(strstr('11,', $err)) {
// $err_msg = "Invalid Transport Protocol";
// }
// elseif(strstr('12,', $err)) {
// $err_msg = "Invalid Transport Protocol";
// }
// elseif(strstr('13,', $err)) {
// $err_msg = "Invalid Transport Protocol";
// }
// elseif(strstr('14,', $err)) {
// $err_msg = "Invalid Transport Protocol";
// }
// elseif(strstr('15,', $err)) {
// $err_msg = "Invalid Transport Protocol";
// }
elseif(strstr($err, '16,')) {
$err_msg = "HTTP error";
}
// ftp error
// elseif(strstr('17,', $err)) {
// $err_msg = "Invalid Transport Protocol";
// }
elseif(strstr($err, '18,')) {
$err_msg = "Unexpected Data(shorter or larger than expected) received.";
}
// ftp error
// elseif(strstr('19,', $err)) {
// $err_msg = "Invalid Transport Protocol";
// }
elseif(strstr($err, '20,')) {
$err_msg = "Downloader Internal Error";
}
// elseif(strstr('21,', $err)) {
// $err_msg = "Invalid Transport Protocol";
// }
elseif(strstr('22,', $err)) {
$err_msg = "Server returned error code >= 400";
}
elseif(strstr($err, '23,')) {
$err_msg = "Downloader Internal Error";
}
elseif(strstr($err, '24,')) {
$err_msg = "Downloader Internal Error";
}
elseif(strstr($err, '25,')) {
$err_msg = "Downloader Internal Error";
}
elseif(strstr($err, '26,')) {
$err_msg = "Downloader Internal Error";
}
elseif(strstr($err, '27,')) {
$err_msg = "Downloader Internal Error";
}
elseif(strstr($err, '28,')) {
$err_msg = "Request timed out.";
}
// ftp error
// elseif(strstr('30,', $err)) {
// $err_msg = "Invalid Transport Protocol";
// }
// elseif(strstr('31,', $err)) {
// $err_msg = "Invalid Transport Protocol";
// }
elseif(strstr($err, '33,')) {
$err_msg = "Chuncked Data not supported by server";
}
elseif(strstr($err, '34,')) {
$err_msg = "Downloader Internal Error";
}
elseif(strstr($err, '35,')) {
$err_msg = "Unable to create SSL handshake.";
}
elseif(strstr($err, '36,')) {
$err_msg = "Resuming page download failed.";
}
elseif(strstr($err, '37,')) {
$err_msg = "Unable to read FILE:// path";
}
elseif(strstr($err, '38,')) {
$err_msg = "LDAP cannot bind";
}
elseif(strstr($err, '39,')) {
$err_msg = "LDAP search failed.";
}
elseif(strstr($err, '41,')) {
$err_msg = "Downloader Internal Error";
}
elseif(strstr($err, '42,')) {
$err_msg = "Downloader Internal Error";
}
elseif(strstr($err, '43,')) {
$err_msg = "Downloader Internal Error";
}
elseif(strstr($err, '44,')) {
$err_msg = "Downloader Internal Error";
}
elseif(strstr($err, '45,')) {
$err_msg = "Downloader Internal Error";
}
elseif(strstr($err, '46,')) {
$err_msg = "Downloader Internal Error";
}
elseif(strstr($err, '47,')) {
$err_msg = "Too many redirects.";
}
elseif(strstr($err, '48,')) {
$err_msg = "Downloader Internal Error";
}
elseif(strstr($err, '49,')) {
$err_msg = "Downloader Internal Error";
}
elseif(strstr($err, '50,')) {
$err_msg = "Downloader Internal Error";
}
elseif(strstr($err, '51,')) {
$err_msg = "The remote server's SSL certificate or SSH md5 fingerprint was deemed not OK. ";
}
elseif(strstr($err, '52,')) {
$err_msg = "Nothing was returned from the server, and under the circumstances, getting nothing is considered an error. ";
}
elseif(strstr($err, '53,')) {
$err_msg = "Downloader Internal Error";
}
elseif(strstr($err, '54,')) {
$err_msg = "Downloader Internal Error";
}
elseif(strstr($err, '55,')) {
$err_msg = "Downloader Internal Error";
}
elseif(strstr($err, '56,')) {
$err_msg = "Failure with receiving network data.";
}
elseif(strstr($err, '57,')) {
$err_msg = "Downloader Internal Error";
}
elseif(strstr($err, '58,')) {
$err_msg = "Downloader Internal Error";
}
elseif(strstr($err, '59,')) {
$err_msg = "Downloader Internal Error";
}
elseif(strstr($err, '60,')) {
$err_msg = "Peer certificate cannot be authenticated with known CA certificates.";
}
elseif(strstr($err, '61,')) {
$err_msg = "Downloader Internal Error";
}
elseif(strstr($err, '62,')) {
$err_msg = "Downloader Internal Error";
}
elseif(strstr($err, '63,')) {
$err_msg = "Maximum file size exceeded.";
}
elseif(strstr($err, '64,')) {
$err_msg = "Requested FTP SSL level failed.";
}
elseif(strstr($err, '65,')) {
$err_msg = "Downloader Internal Error";
}
elseif(strstr($err, '66,')) {
$err_msg = "Downloader Internal Error";
}
elseif(strstr($err, '67,')) {
$err_msg = "The remote server denied login";
}
elseif(strstr($err, '68,')) {
$err_msg = "File not found on TFTP server.";
}
elseif(strstr($err, '69,')) {
$err_msg = "Permission problem on TFTP server.";
}
elseif(strstr($err, '70,')) {
$err_msg = "Out of disk space on the server.";
}
elseif(strstr($err, '71,')) {
$err_msg = "Downloader Internal Error";
}
elseif(strstr($err, '72,')) {
$err_msg = "Downloader Internal Error";
}
elseif(strstr($err, '73,')) {
$err_msg = "Downloader Internal Error";
}
elseif(strstr($err, '74,')) {
$err_msg = "Downloader Internal Error";
}
elseif(strstr($err, '75,')) {
$err_msg = "Downloader Internal Error";
}
elseif(strstr($err, '76,')) {
$err_msg = "Downloader Internal Error";
}
elseif(strstr($err, '77,')) {
$err_msg = "Downloader Internal Error";
}
elseif(strstr($err, '78,')) {
$err_msg = "The resource referenced in the URL does not exist.";
}
elseif(strstr($err, '79,')) {
$err_msg = "Downloader Internal Error";
}
elseif(strstr($err, '80,')) {
$err_msg = "Downloader Internal Error";
}
elseif(strstr($err, '81,')) {
$err_msg = "Downloader Internal Error";
}
elseif(strstr($err, '82,')) {
$err_msg = "Downloader Internal Error";
}
elseif(strstr($err, '83,')) {
$err_msg = "Downloader Internal Error";
}
elseif(strstr($err, '84,')) {
$err_msg = "Downloader Internal Error";
}
elseif(strstr($err, '85,')) {
$err_msg = "Downloader Internal Error";
}
elseif(strstr($err, '86,')) {
$err_msg = "Downloader Internal Error";
}
elseif(strstr($err, '87,')) {
$err_msg = "Downloader Internal Error";
}
elseif(strstr($err, '88,')) {
$err_msg = "Downloader Internal Error";
}
elseif(strstr($err, '89,')) {
$err_msg = "Downloader Internal Error";
}
else {
$err_msg = $err;
}
return $err_msg;
}
public function urlFailed($pages)
{
$fails = array();
$failed = $this->mongoDB->auditor_failed->find(
array('pi' => $this->pid, 'w' => $this->week),
array('_id' => 0, 'u' => 1, 'c' => 1, 'rf' => 1)
);
if(!empty($pages)) {
$failed->limit((int)$pages);
}
while ($ufs = $failed->getNext()) {
$fail = array();
$fail['url'] = urldecode(urldecode(urldecode($ufs['u'])));
$fail['referer'] = urldecode(urldecode(urldecode($ufs['rf'])));
$fail['cause'] = $this->sanitizeCurlFailureCause($ufs['c']);
$fails['failed'][] = $fail;
}
return $fails;
}
public function urlNon200($pages)
{
$fails = array();
$failed = $this->mongoDB->auditor_non200->find(
array('pi' => $this->pid, 'w' => $this->week),
array('_id' => 0, 'u' => 1, 's' => 1, 'rf' => 1)
);
if(!empty($pages)) {
$failed->limit((int)$pages);
}
while ($ufs = $failed->getNext()) {
$fail = array();
$fail['stat'] = $ufs['s'];
$fail['url'] = urldecode(urldecode(urldecode($ufs['u'])));
$fail['referer'] = urldecode(urldecode(urldecode($ufs['rf'])));
$fails['non200'][] = $fail;
}
return $fails;
}
public function urlNonIndex($pages)
{
$noni = array();
$failed = $this->mongoDB->auditor_nonindex->find(
array('pi' => $this->pid, 'w' => $this->week),
array('_id' => 0, 'u' => 1, 'ca' => 1, 'rf' => 1)
);
if(!empty($pages)) {
$failed->limit((int)$pages);
}
while ($ufs = $failed->getNext()) {
$nonc = array();
$nonc['url'] = urldecode(urldecode(urldecode($ufs['u'])));
$nonc['referer'] = urldecode(urldecode(urldecode($ufs['rf'])));
if(isset($ufs['ca']) && $ufs['ca'] == 1) {
$noni['canonical'][] = $nonc;
} else {
$noni['nonindex'][] = $nonc;
}
}
return $noni;
}
public function urlUnique($pages)
{
$fails = array();
$failed = $this->mongoDB->auditor->find(
array('pi' => $this->pid, 'w' => $this->week, 'tu' => null),
array('_id' => 0, 'u' => 1)
);
if(!empty($pages)) {
$failed->limit((int)$pages);
}
while ($ufs = $failed->getNext()) {
$fails['url'][] = urldecode(urldecode(urldecode($ufs['u'])));
}
return $fails;
}
public function title_length()
{
$data = $this->mongoDB->auditor->findOne(
array('pi' => $this->pid, 'w' => $this->week, 'pos' => $this->pos),
array('_id' => 0, 'etit.tit' => 1)
);
$title['Length'] = strlen($data['etit']['tit']);
$title['Title'] = $data['etit']['tit'];
return $title;
}
public function title_duplicate($count_only=null)
{
$data = $this->mongoDB->auditor->findOne(
array('pi' => $this->pid, 'w' => $this->week, 'pos' => $this->pos),
array('_id' => 0, 'etit.tit' => 1)
);
if(!empty($count_only))
{
$dup_title_count = $this->mongoDB->auditor->count(
array('pi' => $this->pid, 'w' => $this->week, 'etit.tit' => $data['etit']['tit'])
);
return $dup_title_count;
}
$title['Length'] = strlen($data['etit']['tit']);
$title['Title'] = $data['etit']['tit'];
$dup_title = $this->mongoDB->auditor->find(
array('pi' => $this->pid, 'w' => $this->week, 'etit.tit' => $data['etit']['tit']),
array('_id' => 0, 'u' => 1)
);
while ($dups = $dup_title->getNext()) {
if(!empty($dups['u'])) {
$title['url'][] = urldecode(urldecode(urldecode($dups['u'])));
}
}
return $title;
}
public function meta_kw_length()
{
$data = $this->mongoDB->auditor->findOne(
array('pi' => $this->pid, 'w' => $this->week, 'pos' => $this->pos),
array('_id' => 0, 'emet.kws' => 1)
);
$meta['Length'] = strlen($data['emet']['kws']);
$meta['Keyword'] = $data['emet']['kws'];
return $meta;
}
public function missing_alt()
{
$alts_urls = array();
$missing_alt_arr = array();
$miss_alt = $this->mongoDB->auditor->findOne(
array('pi' => $this->pid, 'w' => $this->week, 'pos'=>$this->pos, 'eimg.mal' => 1),
array('_id' => 0, 'eimg.mah' => 1)
);
// while ($m_alts = $miss_alt->getNext()) {
// $missing_alt_arr[] = $m_alts['eimg']['mah'];
// }
foreach ($miss_alt['eimg']['mah'] as $value) {
if(!empty($value)) {
$href = rel2abs($value, "http://".$this->domain);
// $resized_href = imagejpeg(resize_image($href,20, 20), 'image.jpg');
// $href_ico = "![](".$resized_href.")
";
$alts_urls['Image HREFs Which Have Missing ALT Declaration'][] = $href;
}
}
return $alts_urls;
}
public function meta_kw_duplicate($count_only=null)
{
$data = $this->mongoDB->auditor->findOne(
array('pi' => $this->pid, 'w' => $this->week, 'pos' => $this->pos),
array('_id' => 0, 'emet.kws' => 1)
);
if(!empty($count_only))
{
$dup_meta_kw_count = $this->mongoDB->auditor->count(
array('pi' => $this->pid, 'w' => $this->week, 'emet.kws' => $data['emet']['kws'])
);
return $dup_meta_kw_count;
}
$meta['Length'] = strlen($data['emet']['kws']);
$meta['Keyword'] = $data['emet']['kws'];
$dup_meta_kw = $this->mongoDB->auditor->find(
array('pi' => $this->pid, 'w' => $this->week, 'emet.kws' => $data['emet']['kws']),
array('_id' => 0, 'u' => 1)
);
while ($dups = $dup_meta_kw->getNext()) {
$meta['url'][] = urldecode(urldecode(urldecode($dups['u'])));
}
return $meta;
}
public function meta_desc_length()
{
$data = $this->mongoDB->auditor->findOne(
array('pi' => $this->pid, 'w' => $this->week, 'pos' => $this->pos),
array('_id' => 0, 'emet.desc' => 1)
);
$meta['Length'] = strlen($data['emet']['desc']);
$meta['Description'] = $data['emet']['desc'];
return $meta;
}
public function meta_desc_duplicate($count_only=null)
{
$data = $this->mongoDB->auditor->findOne(
array('pi' => $this->pid, 'w' => $this->week, 'pos' => $this->pos),
array('_id' => 0, 'emet.desc' => 1)
);
if(!empty($count_only))
{
$dup_meta_desc_count = $this->mongoDB->auditor->count(
array('pi' => $this->pid, 'w' => $this->week, 'emet.desc' => $data['emet']['desc'])
);
return $dup_meta_desc_count;
}
$meta['Length'] = strlen($data['emet']['desc']);
$meta['Description'] = $data['emet']['desc'];
$dup_meta_desc = $this->mongoDB->auditor->find(
array('pi' => $this->pid, 'w' => $this->week, 'emet.desc' => $data['emet']['desc']),
array('_id' => 0, 'u' => 1)
);
while ($dups = $dup_meta_desc->getNext()) {
$meta['url'][] = urldecode(urldecode(urldecode($dups['u'])));
}
return $meta;
}
public function url_length()
{
$data = $this->mongoDB->auditor->findOne(
array('pi' => $this->pid, 'w' => $this->week, 'pos' => $this->pos),
array('_id' => 0, 'u' => 1)
);
$url['Length'] = strlen($data['u']);
$url['url'] = urldecode(urldecode(urldecode($data['u'])));
return $url;
}
public function links_int()
{
$data = $this->mongoDB->auditor->findOne(
array('pi' => $this->pid, 'w' => $this->week, 'pos' => $this->pos),
array('_id' => 0, "eln.nf.int" => 1, "eln.f.int" => 1, "eln.int" => 1)
);
if(!empty($data['eln']['f']['int'])) {
foreach ($data['eln']['f']['int'] as $value) {
$value = urldecode(urldecode(urldecode($value)));
$f_arr[] = rel2abs($value, "http://".$this->domain);
}
}
if(!empty($data['eln']['nf']['int'])) {
foreach ($data['eln']['nf']['int'] as $nf_value) {
$nf_value = urldecode(urldecode(urldecode($nf_value)));
$nf_arr[] = rel2abs($nf_value, "http://".$this->domain);
}
}
if(!empty($data['eln']['int'])) $links['Count'] = $data['eln']['int'];
if(!empty($f_arr)) $links['Follow'] = $f_arr;
if(!empty($nf_arr)) $links['No Follow'] = $nf_arr;
return $links;
}
public function links_ext()
{
$data = $this->mongoDB->auditor->findOne(
array('pi' => $this->pid, 'w' => $this->week, 'pos' => $this->pos),
array('_id' => 0, "eln.nf.ext" => 1, "eln.f.ext" => 1, "eln.ext" => 1)
);
if(!empty($data['eln']['f']['ext'])) {
foreach ($data['eln']['f']['ext'] as $value) {
$value = urldecode(urldecode(urldecode($value)));
$f_arr[] = rel2abs($value, "http://".$this->domain);
}
}
if(!empty($data['eln']['nf']['ext'])) {
foreach ($data['eln']['nf']['ext'] as $nf_value) {
$nf_value = urldecode(urldecode(urldecode($nf_value)));
$nf_arr[] = rel2abs($nf_value, "http://".$this->domain);
}
}
if(!empty($data['eln']['ext'])) $links['count'] = $data['eln']['ext'];
if(!empty($data['eln']['f']['ext'])) $links['follow'] = $f_arr;
if(!empty($data['eln']['nf']['ext'])) $links['nofollow'] = $nf_arr;
return $links;
}
public function links_int_nofollow()
{
$data = $this->mongoDB->auditor->findOne(
array('pi' => $this->pid, 'w' => $this->week, 'pos' => $this->pos),
array('_id' => 0, "eln.nf.int" => 1)
);
$links['Length'] = count($data['eln']['nf']['int']);
$links['Links'] = $data['eln']['nf']['int'];
return $links;
}
public function links_ext_nofollow()
{
$data = $this->mongoDB->auditor->findOne(
array('pi' => $this->pid, 'w' => $this->week, 'pos' => $this->pos),
array('_id' => 0, "eln.nf.ext" => 1)
);
$links['Length'] = count($data['eln']['nf']['ext']);
$links['Links'] = $data['eln']['nf']['ext'];
return $links;
}
public function links_int_follow()
{
$data = $this->mongoDB->auditor->findOne(
array('pi' => $this->pid, 'w' => $this->week, 'pos' => $this->pos),
array('_id' => 0, "eln.f.int" => 1)
);
$links['Length'] = count($data['eln']['f']['int']);
$links['Links'] = $data['eln']['f']['int'];
return $links;
}
public function links_ext_follow()
{
$data = $this->mongoDB->auditor->findOne(
array('pi' => $this->pid, 'w' => $this->week, 'pos' => $this->pos),
array('_id' => 0, "eln.f.ext" => 1)
);
$links['Length'] = count($data['eln']['f']['ext']);
$links['Links'] = $data['eln']['f']['ext'];
return $links;
}
public function content_duplicate()
{
$final_arr = array();
$dup_content = $this->mongoDB->auditor->findOne(
array('pi' => $this->pid, 'w' => $this->week, 'tu' => array('$gt' => 0), 'pos'=>$this->pos),
array('_id' => 0, 'u' => 1, 'dw' => 1)
);
$all_urls = $this->mongoDB->auditor->find(
array('pi' => $this->pid, 'w' => $this->week),
array('_id' => 0, 'u' => 1, 'pos' => 1)
);
while ($urls = $all_urls->getNext()) {
$urls_arr[$urls['pos']] = urldecode(urldecode(urldecode($urls['u'])));
}
$dup_con = array();
$dup_con['url'] = urldecode(urldecode(urldecode($dup_content['u'])));
foreach ($dup_content['dw'] as $key => $value) {
$final_arr[$dup_con['url']]['Duplicates'][$urls_arr[$value['pos']]] = $value['perc'];
}
return $final_arr;
}
public function img_many()
{
$many_img = $this->mongoDB->auditor->findOne(
array('pi' => $this->pid, 'w' => $this->week, 'pos' => $this->pos),
array('_id' => 0, 'u' => 1, 'eimg.hrf' => 1, 'eimg.c' => 1)
);
$final_arr['Count'] = $many_img['eimg']['c'];
foreach ($many_img['eimg']['hrf'] as $e_href) {
$img_arr = array();
$href = rel2abs($e_href, "http://".$this->domain);
// $href_ico = "![](".$href.")
";
$final_arr['Image HREFs'][] = $href;
}
return $final_arr;
}
public function sheets_many()
{
$many_img = $this->mongoDB->auditor->findOne(
array('pi' => $this->pid, 'w' => $this->week, 'pos' => $this->pos),
array('_id' => 0, 'u' => 1, 'esht.sts' => 1)
);
$final_arr['Count'] = count($many_img['esht']['sts']);
$final_arr['Styles Sheets'] = $many_img['esht']['sts'];
return $final_arr;
}
public function spell_mistakes()
{
$many_img = $this->mongoDB->auditor->findOne(
array('pi' => $this->pid, 'w' => $this->week, 'pos' => $this->pos),
array('_id' => 0, 'u' => 1, 'espl.mis' => 1)
);
$final_arr['Count'] = count($many_img['espl']['mis']);
$final_arr['Mistakes'] = $many_img['espl']['mis'];
return $final_arr;
}
public function many_h1()
{
$many_h1 = $this->mongoDB->auditor->findOne(
array('pi' => $this->pid, 'w' => $this->week, 'pos' => $this->pos),
array('_id' => 0, 'u' => 1, 'eh1s.h1t' => 1)
);
$final_arr['H1 Tags'] = $many_h1['eh1s']['h1t'];
return $final_arr;
}
public function many_h2()
{
$many_h2 = $this->mongoDB->auditor->findOne(
array('pi' => $this->pid, 'w' => $this->week, 'pos' => $this->pos),
array('_id' => 0, 'u' => 1, 'eh2s.h2t' => 1)
);
$final_arr['H2 Tags'] = $many_h2['eh2s']['h2t'];
return $final_arr;
}
}
?>