<?php


    require_once "conf.php"; //common configuration constants and error function defined

    class ErrorDetail
    {
        private $domain;
        private $pid;
        private $pos;
        private $week;
        private $mongoDB;
        private $limit;
        private $offset;
        private $past_week;

        public function __construct($p_id, $project="", $url_pos="",$ca_id= 0,$pa_id= 0, $limit= 50, $offset = 0 )
        {
            $mongo = new mongoClient(MONGO_SERVER);
            $this->mongoDB = $mongo->selectDB("distributed");
            $this->pid = $p_id;
            if($project !== "") {
                $this->domain = $project;
            }
            if($url_pos !== "") {
                $this->pos = (int)$url_pos;
            }
            if($ca_id !== "") {
                $this->week = (int)$ca_id;
            }
            if($pa_id !== "") {
                $this->past_week = (int)$pa_id;
            }
            $this->limit = $limit;
            $this->offset = $offset;
        }

        public function sanitizeCurlFailureCause($err)
        {
            $err_msg = "";
            if(strstr($err, '1,')) {
                $err_msg = "Invalid Transport Protocol";
            }
            elseif(strstr($err, '2,')) {
                $err_msg = "Downloader Internal Error";
            }
            elseif(strstr($err, '3,')) {
                $err_msg = "The URL was not properly formatted.";
            }
            elseif(strstr($err, '4,')) {
                $err_msg = "Downloader Internal Error";
            }
            elseif(strstr($err, '5,')) {
                $err_msg = "Unable resolve proxy.";
            }
            elseif(strstr($err, '6,')) {
                $err_msg = "Unable to resolve Host.";
            }
            elseif(strstr($err, '7,')) {
                $err_msg = "Unable to connect to host";
            }
            elseif(strstr($err, '8,')) {
                $err_msg = "Bad response from server";
            }
            elseif(strstr('9,', $err)) {
                $err_msg = "Access denied on server";
            }
            // ftp errors
            // elseif(strstr('10,', $err)) {
            //     $err_msg = "Invalid Transport Protocol";
            // }
            // elseif(strstr('11,', $err)) {
            //     $err_msg = "Invalid Transport Protocol";
            // }
            // elseif(strstr('12,', $err)) {
            //     $err_msg = "Invalid Transport Protocol";
            // }
            // elseif(strstr('13,', $err)) {
            //     $err_msg = "Invalid Transport Protocol";
            // }
            // elseif(strstr('14,', $err)) {
            //     $err_msg = "Invalid Transport Protocol";
            // }
            // elseif(strstr('15,', $err)) {
            //     $err_msg = "Invalid Transport Protocol";
            // }
            elseif(strstr($err, '16,')) {
                $err_msg = "HTTP error";
            }
            // ftp error
            // elseif(strstr('17,', $err)) {
            //     $err_msg = "Invalid Transport Protocol";
            // }
            elseif(strstr($err, '18,')) {
                $err_msg = "Unexpected Data(shorter or larger than expected) received.";
            }
            // ftp error
            // elseif(strstr('19,', $err)) {
            //     $err_msg = "Invalid Transport Protocol";
            // }
            elseif(strstr($err, '20,')) {
                $err_msg = "Downloader Internal Error";
            }
            // elseif(strstr('21,', $err)) {
            //     $err_msg = "Invalid Transport Protocol";
            // }
            elseif(strstr('22,', $err)) {
                $err_msg = "Server returned error code >= 400";
            }
            elseif(strstr($err, '23,')) {
                $err_msg = "Downloader Internal Error";
            }
            elseif(strstr($err, '24,')) {
                $err_msg = "Downloader Internal Error";
            }
            elseif(strstr($err, '25,')) {
                $err_msg = "Downloader Internal Error";
            }
            elseif(strstr($err, '26,')) {
                $err_msg = "Downloader Internal Error";
            }
            elseif(strstr($err, '27,')) {
                $err_msg = "Downloader Internal Error";
            }
            elseif(strstr($err, '28,')) {
                $err_msg = "Request timed out.";
            }
            // ftp error
            // elseif(strstr('30,', $err)) {
            //     $err_msg = "Invalid Transport Protocol";
            // }
            // elseif(strstr('31,', $err)) {
            //     $err_msg = "Invalid Transport Protocol";
            // }
            elseif(strstr($err, '33,')) {
                $err_msg = "Chuncked Data not supported by server";
            }
            elseif(strstr($err, '34,')) {
                $err_msg = "Downloader Internal Error";
            }
            elseif(strstr($err, '35,')) {
                $err_msg = "Unable to create SSL handshake.";
            }
            elseif(strstr($err, '36,')) {
                $err_msg = "Resuming page download failed.";
            }
            elseif(strstr($err, '37,')) {
                $err_msg = "Unable to read FILE:// path";
            }
            elseif(strstr($err, '38,')) {
                $err_msg = "LDAP cannot bind";
            }
            elseif(strstr($err, '39,')) {
                $err_msg = "LDAP search failed.";
            }
            elseif(strstr($err, '41,')) {
                $err_msg = "Downloader Internal Error";
            }
            elseif(strstr($err, '42,')) {
                $err_msg = "Downloader Internal Error";
            }
            elseif(strstr($err, '43,')) {
                $err_msg = "Downloader Internal Error";
            }
            elseif(strstr($err, '44,')) {
                $err_msg = "Downloader Internal Error";
            }
            elseif(strstr($err, '45,')) {
                $err_msg = "Downloader Internal Error";
            }
            elseif(strstr($err, '46,')) {
                $err_msg = "Downloader Internal Error";
            }
            elseif(strstr($err, '47,')) {
                $err_msg = "Too many redirects.";
            }
            elseif(strstr($err, '48,')) {
                $err_msg = "Downloader Internal Error";
            }
            elseif(strstr($err, '49,')) {
                $err_msg = "Downloader Internal Error";
            }
            elseif(strstr($err, '50,')) {
                $err_msg = "Downloader Internal Error";
            }
            elseif(strstr($err, '51,')) {
                $err_msg = "The remote server's SSL certificate or SSH md5 fingerprint was deemed not OK. ";
            }
            elseif(strstr($err, '52,')) {
                $err_msg = "Nothing was returned from the server, and under the circumstances, getting nothing is considered an error. ";
            }
            elseif(strstr($err, '53,')) {
                $err_msg = "Downloader Internal Error";
            }
            elseif(strstr($err, '54,')) {
                $err_msg = "Downloader Internal Error";
            }
            elseif(strstr($err, '55,')) {
                $err_msg = "Downloader Internal Error";
            }
            elseif(strstr($err, '56,')) {
                $err_msg = "Failure with receiving network data.";
            }
            elseif(strstr($err, '57,')) {
                $err_msg = "Downloader Internal Error";
            }
            elseif(strstr($err, '58,')) {
                $err_msg = "Downloader Internal Error";
            }
            elseif(strstr($err, '59,')) {
                $err_msg = "Downloader Internal Error";
            }
            elseif(strstr($err, '60,')) {
                $err_msg = "Peer certificate cannot be authenticated with known CA certificates.";
            }
            elseif(strstr($err, '61,')) {
                $err_msg = "Downloader Internal Error";
            }
            elseif(strstr($err, '62,')) {
                $err_msg = "Downloader Internal Error";
            }
            elseif(strstr($err, '63,')) {
                $err_msg = "Maximum file size exceeded.";
            }
            elseif(strstr($err, '64,')) {
                $err_msg = "Requested FTP SSL level failed.";
            }
            elseif(strstr($err, '65,')) {
                $err_msg = "Downloader Internal Error";
            }
            elseif(strstr($err, '66,')) {
                $err_msg = "Downloader Internal Error";
            }
            elseif(strstr($err, '67,')) {
                $err_msg = "The remote server denied login";
            }
            elseif(strstr($err, '68,')) {
                $err_msg = "File not found on TFTP server.";
            }
            elseif(strstr($err, '69,')) {
                $err_msg = "Permission problem on TFTP server.";
            }
            elseif(strstr($err, '70,')) {
                $err_msg = "Out of disk space on the server.";
            }
            elseif(strstr($err, '71,')) {
                $err_msg = "Downloader Internal Error";
            }
            elseif(strstr($err, '72,')) {
                $err_msg = "Downloader Internal Error";
            }
            elseif(strstr($err, '73,')) {
                $err_msg = "Downloader Internal Error";
            }
            elseif(strstr($err, '74,')) {
                $err_msg = "Downloader Internal Error";
            }
            elseif(strstr($err, '75,')) {
                $err_msg = "Downloader Internal Error";
            }
            elseif(strstr($err, '76,')) {
                $err_msg = "Downloader Internal Error";
            }
            elseif(strstr($err, '77,')) {
                $err_msg = "Downloader Internal Error";
            }
            elseif(strstr($err, '78,')) {
                $err_msg = "The resource referenced in the URL does not exist.";
            }
            elseif(strstr($err, '79,')) {
                $err_msg = "Downloader Internal Error";
            }
            elseif(strstr($err, '80,')) {
                $err_msg = "Downloader Internal Error";
            }
            elseif(strstr($err, '81,')) {
                $err_msg = "Downloader Internal Error";
            }
            elseif(strstr($err, '82,')) {
                $err_msg = "Downloader Internal Error";
            }
            elseif(strstr($err, '83,')) {
                $err_msg = "Downloader Internal Error";
            }
            elseif(strstr($err, '84,')) {
                $err_msg = "Downloader Internal Error";
            }
            elseif(strstr($err, '85,')) {
                $err_msg = "Downloader Internal Error";
            }
            elseif(strstr($err, '86,')) {
                $err_msg = "Downloader Internal Error";
            }
            elseif(strstr($err, '87,')) {
                $err_msg = "Downloader Internal Error";
            }
            elseif(strstr($err, '88,')) {
                $err_msg = "Downloader Internal Error";
            }
            elseif(strstr($err, '89,')) {
                $err_msg = "Downloader Internal Error";
            }
            else {
                $err_msg = $err;
            }

            return $err_msg;
        }

        public function urlFailed()
        {
            $fails = array();
            $failed = $this->mongoDB->auditor_status->find(
                array('p_id' => $this->pid, 'a_id' => $this->week, 'status'=>1001),
                array('_id' => 0, 'url' => 1, 'u' => 1 ,'err'=>1, 'rf' => 1, 'referer'=>1,'src'=>1)
            )->skip($this->offset)->limit($this->limit);

            while ($ufs = $failed->getNext()) {
                $fail = array();
                $ufs['url'] = (isset($ufs['url'])) ? $ufs['url'] : $ufs['u'];
                $ufs['rf'] = (isset($ufs['rf'])) ? $ufs['rf'] : $ufs['referer'];
                $fail['url'] = urldecode(urldecode(urldecode($ufs['url'])));
                $fail['referer'] = urldecode(urldecode(urldecode($ufs['rf'])));
                $fail['cause'] = $this->sanitizeCurlFailureCause($ufs['err']);
                $fails['failed'][] = $fail;
                $fails['src'] = (isset($ufs['src']))? $ufs['src'] : '';
            }
            $total_pages = $this->mongoDB->auditor_status->count(
                array('p_id' => $this->pid, 'a_id' => $this->week, 'status'=>1001)
            );
            $fails['total_pages'] = $total_pages;
            return $fails;
        }

        public function urlNon200()
        {
            $fails = array();

            $failed = $this->mongoDB->auditor_status->find(
                array('p_id' => $this->pid, 'a_id' => $this->week,'$or' => 
                    array(
                        array("status" => array('$lt'=>200)),
                        array("status" => array('$gt'=>400,'$lt'=>1000))
                    )
                ),
                array('_id' => 0, 'url' => 1,'u' => 1, 'status' => 1, 'rf' => 1,'referer' => 1,'src'=>1)
            )->skip($this->offset)->limit($this->limit);

            while ($ufs = $failed->getNext()) {
                $fail = array();
                $fail['stat'] = $ufs['status'];
                $ufs['url'] = (isset($ufs['url'])) ? $ufs['url'] : $ufs['u'];
                $ufs['rf'] = (isset($ufs['rf'])) ? $ufs['rf'] : $ufs['referer'];
                $fail['url'] = urldecode(urldecode(urldecode($ufs['url'])));
                $fail['referer'] = urldecode(urldecode(urldecode($ufs['rf'])));
                $fails['non200'][] = $fail;
                $fails['src'] = (isset($ufs['src']))? $ufs['src'] : '';
            }
            $total_pages = $this->mongoDB->auditor_status->count(array('p_id' => $this->pid, 'a_id' => $this->week,'$or' => 
                    array(
                        array("status" => array('$lt'=>200)),
                        array("status" => array('$gt'=>400,'$lt'=>1000))
                    )));
            $fails['total_pages'] = $total_pages;
            return $fails;
        }

        public function urlNonIndex()
        {
            $noni = array();

            $failed = $this->mongoDB->auditor_status->find(
                array('p_id' => $this->pid, 'a_id' => $this->week,'status'=>1000),
                array('_id' => 0, 'url' => 1, 'u' => 1,'ca' => 1, 'rf' => 1, 'referer' => 1,'src'=>1)
            )->skip($this->offset)->limit($this->limit);

            while ($ufs = $failed->getNext()) {
                $nonc = array();
                $ufs['url'] = (isset($ufs['url'])) ? $ufs['url'] : $ufs['u'];
                $ufs['rf'] = (isset($ufs['rf'])) ? $ufs['rf'] : $ufs['referer'];
                $nonc['url'] = urldecode(urldecode(urldecode($ufs['url'])));
                $nonc['referer'] = urldecode(urldecode(urldecode($ufs['rf'])));
                $noni['src'] = (isset($ufs['src']))? $ufs['src'] : '';
                if(isset($ufs['ca']) && $ufs['ca'] == 1) {
                    $noni['canonical'][] = $nonc;
                } else {
                    $noni['nonindex'][] = $nonc;
                }
            }
            $total_pages = $this->mongoDB->auditor_status->count(
                array('p_id' => $this->pid, 'a_id' => $this->week,'status'=>1000)
            );
            $noni['total_pages'] = $total_pages;
            return $noni;
        }

        public function urlUnique()
        {
            $fails = array();
            $failed = $this->mongoDB->auditor->find(
                array('p_id' => $this->pid, 'a_id' => $this->week),
                array('_id' => 0, 'url' => 1,'u' => 1,'src'=>1)
            )->skip($this->offset)->limit($this->limit);

            while ($ufs = $failed->getNext()) {
                $ufs['url'] = (isset($ufs['url'])) ? $ufs['url'] : $ufs['u'];
                $fails['url'][urldecode(urldecode(urldecode($ufs['url'])))] = isset($ufs['src']) ? $ufs['src'] : '';
            }
            $total_pages = $this->mongoDB->auditor->count(array('p_id' => $this->pid, 'a_id' => $this->week));
            $fails['total_pages'] = $total_pages;
            return $fails;
        }

        public function urlOrphaned()
        {
            $fails = array();
            try{
                $failed = $this->mongoDB->orphaned_urls->find(
                array('p_id' => $this->pid, 'a_id' => $this->week),
                array('_id' => 0, 'orp_urls' => 1)
                );
                $fails = $failed->getNext();
            }
            catch(Exception $e){
                echo $e->getMessage();
            }
            return $fails;
        }

        public function urlCanonical()
        {
            $canonical_urls = array();
            try{
                $can_data = $this->mongoDB->au_canonical->find(
                array('p_id' => $this->pid, 'a_id' => $this->week),
                array('_id' => 0, 'url' => 1,'rd_from'=>1,'rf'=>1,'depth'=>1)
                );
            }
            catch(Exception $e){
                echo $e->getMessage();
            }
            while ($data = $can_data->getNext()) {
                $ref = (isset($data['rf'])) ? $data['rf'] : $data['rd_from'];
                $canonical_urls[$data['rd_from']][] = array('can'=>$data['url'],'referer'=>$ref,'depth'=>$data['depth']);
            }
            return $canonical_urls;
        }

        public function urlRedirected()
        {
            $redirected_urls = array();
            try{
                $can_data = $this->mongoDB->au_redirected->find(
                array('p_id' => $this->pid, 'a_id' => $this->week),
                array('_id' => 0, 'url' => 1,'rd_from'=>1,'rf'=>1,'depth'=>1)
                );
            }
            catch(Exception $e){
                echo $e->getMessage();
            }
            while ($data = $can_data->getNext()) {
                $ref = (isset($data['rf'])) ? $data['rf'] : $data['rd_from'];
                $redirected_urls[$data['rd_from']][] = array('can'=>$data['url'],'referer'=>$ref,'depth'=>$data['depth']);
            }
            return $redirected_urls;
        }

        public function urlFailedPast($pages, $last_checked_on)
        {
            $final_arr = array();
            $url_cur = array();
            $url_past = array();
            $fails = array();
            $fails_past = array();

            $failed = $this->mongoDB->auditor_status->find(
                array('p_id' => $this->pid, 'a_id' => $this->week,'status'=> 1001),
                array('_id' => 0, 'url' => 1, 'c' => 1, 'rf' => 1, 'u'=>1 ,'referer' =>1,'src'=>1)
            );
            if(!empty($pages)) {
                $failed->limit((int)$pages);
            }

            while ($ufs = $failed->getNext()) {
                $ufs['url'] = (isset($ufs['url'])) ? $ufs['url'] : $ufs['u'];
                $ufs['rf'] = (isset($ufs['rf'])) ? $ufs['rf'] : $ufs['referer'];
                $url = trim(strip_tags(urldecode(urldecode(urldecode($ufs['url'])))));
                $url_cur[] = $url;
                $fails[$url]['url'] = $url;
                $fails[$url]['referer'] = urldecode(urldecode(urldecode($ufs['rf'])));
                $fails[$url]['cause'] = $this->sanitizeCurlFailureCause($ufs['c']);
            }

            $failed_p = $this->mongoDB->auditor_failed->find(
                array('p_id' => $this->pid, 'a_id' => $this->past_week),
                array('_id' => 0, 'url' => 1, 'c' => 1, 'rf' => 1,'u'=>1,'referer'=>1)
            );
            if(!empty($pages)) {
                $failed_p->limit((int)$pages);
            }
            while ($ufs2 = $failed_p->getNext()) {
                $ufs2['url'] = (isset($ufs2['url'])) ? $ufs2['url'] : $ufs2['u'];
                $ufs2['rf'] = (isset($ufs2['rf'])) ? $ufs2['rf'] : $ufs2['referer'];
                $url2 = trim(strip_tags(urldecode(urldecode(urldecode($ufs2['url'])))));
                $url_past[] = $url2;
                $fails_past[$url2]['url'] = $url2;
                $fails_past[$url2]['referer'] = urldecode(urldecode(urldecode($ufs2['rf'])));
                $fails_past[$url2]['cause'] = $this->sanitizeCurlFailureCause($ufs2['c']);
            }

            $url_new = array_diff($url_cur, $url_past);
            $url_deleted = array_diff($url_past, $url_cur);

            foreach ($url_new as $unvalue) {
                $final_arr['failed']['new'][] = $fails[$unvalue];
            }
            foreach ($url_deleted as $udvalue) {
                $final_arr['failed']['deleted'][] = $fails_past[$udvalue];
            }
            return $final_arr;
        }

        public function urlNon200Past($pages,$last_checked_on)
        {
            $final_arr = array();
            $url_cur = array();
            $url_past = array();
            $fails = array();
            $fails_past = array();

            $failed = $this->mongoDB->auditor_status->find(
                array('p_id' => $this->pid, 'a_id' => $this->week),
                array('_id' => 0, 'url' => 1, 's' => 1, 'rf' => 1, 'u' => 1,'referer' => 1,'src'=>1)
            );
            if(!empty($pages)) {
                $failed->limit((int)$pages);
            }

            while ($ufs = $failed->getNext()) {
                $ufs['url'] = (isset($ufs['url'])) ? $ufs['url'] : $ufs['u'];
                $ufs['rf'] = (isset($ufs['rf'])) ? $ufs['rf'] : $ufs['referer'];
                $url = trim(strip_tags(urldecode(urldecode(urldecode($ufs['url'])))));
                $url_cur[] = $url;
                $fails[$url]['url'] = $url;
                $fails[$url]['referer'] = urldecode(urldecode(urldecode($ufs['rf'])));
                $fails[$url]['stat'] = $ufs['s'];
            }

            $failed_p = $this->mongoDB->auditor_status->find(
                array('p_id' => $this->pid, 'a_id' => $this->past_week),
                array('_id' => 0, 'url' => 1, 's' => 1, 'rf' => 1, 'u' => 1 ,'referer' => 1)
            );
            if(!empty($pages)) {
                $failed_p->limit((int)$pages);
            }
            while ($ufs2 = $failed_p->getNext()) {
                $ufs2['url'] = (isset($ufs2['url'])) ? $ufs2['url'] : $ufs2['u'];
                $ufs2['rf'] = (isset($ufs2['rf'])) ? $ufs2['rf'] : $ufs2['referer'];
                $url2 = trim(strip_tags(urldecode(urldecode(urldecode($ufs2['url'])))));
                $url_past[] = $url2;
                $fails_past[$url2]['url'] = $url2;
                $fails_past[$url2]['referer'] = urldecode(urldecode(urldecode($ufs2['rf'])));
                $fails_past[$url2]['stat'] = $ufs2['s'];
            }

            $url_new = array_diff($url_cur, $url_past);
            $url_deleted = array_diff($url_past, $url_cur);

            foreach ($url_new as $unvalue) {
                $final_arr['non200']['new'][] = $fails[$unvalue];
            }
            foreach ($url_deleted as $udvalue) {
                $final_arr['non200']['deleted'][] = $fails_past[$udvalue];
            }
            return $final_arr;
        }


        public function urlNonIndexPast($pages,$last_checked_on)
        {
            $final_arr = array();
            $url_cur = array();
            $url_past = array();
            $fails = array();
            $fails_past = array();

            $failed = $this->mongoDB->auditor_status->find(
                array('p_id' => $this->pid, 'a_id' => $this->week),
                array('_id' => 0, 'url' => 1, 'ca' => 1, 'rf' => 1 , 'u'=>1 ,'referer' => 1,'src'=>1)
            );
            if(!empty($pages)) {
                $failed->limit((int)$pages);
            }

            while ($ufs = $failed->getNext()) {
                $ufs['url'] = (isset($ufs['url'])) ? $ufs['url'] : $ufs['u'];
                $ufs['rf'] = (isset($ufs['rf'])) ? $ufs['rf'] : $ufs['referer'];
                $url = trim(strip_tags(urldecode(urldecode(urldecode($ufs['url'])))));
                $url_cur[] = $url;
                $fails[$url]['url'] = $url;
                $fails[$url]['referer'] = urldecode(urldecode(urldecode($ufs['rf'])));
                if(isset($ufs['ca']) && $ufs['ca'] == 1) {
                    $fails[$url]['ca'] = 1;
                } else {
                    $fails[$url]['ca'] = 0;
                }

            }

            $failed_p = $this->mongoDB->auditor_status->find(
                array('p_id' => $this->pid, 'a_id' => $this->past_week),
                array('_id' => 0, 'url' => 1, 'ca' => 1, 'rf' => 1, 'u'=>1, 'referer' => 1)
            );
            if(!empty($pages)) {
                $failed_p->limit((int)$pages);
            }
            while ($ufs2 = $failed_p->getNext()) {
                $ufs2['url'] = (isset($ufs2['url'])) ? $ufs2['url'] : $ufs2['u'];
                $ufs2['rf'] = (isset($ufs2['rf'])) ? $ufs2['rf'] : $ufs2['referer'];
                $url2 = trim(strip_tags(urldecode(urldecode(urldecode($ufs2['url'])))));
                $url_past[] = $url2;
                $fails_past[$url2]['url'] = $url2;
                $fails_past[$url2]['referer'] = urldecode(urldecode(urldecode($ufs2['rf'])));
                if(isset($ufs2['ca']) && $ufs2['ca'] == 1) {
                    $fails_past[$url2]['ca'] = 1;
                } else {
                    $fails_past[$url2]['ca'] = 0;
                }
            }

            $url_new = array_diff($url_cur, $url_past);
            $url_deleted = array_diff($url_past, $url_cur);

            foreach ($url_new as $unvalue) {
                if($fails[$unvalue]['ca'] == 1) {
                    $final_arr['canonical']['new'][] = $fails[$unvalue];
                } elseif($fails[$unvalue]['ca'] == 0) {
                    $final_arr['nonindex']['new'][] = $fails[$unvalue];
                }
            }
            foreach ($url_deleted as $udvalue) {
                if($fails_past[$udvalue]['ca'] == 1) {
                    $final_arr['canonical']['deleted'][] = $fails_past[$udvalue];
                } elseif($fails_past[$udvalue]['ca'] == 0) {
                    $final_arr['nonindex']['deleted'][] = $fails_past[$udvalue];
                }
            }
            return $final_arr;
        }


        public function urlUniquePast($pages,$last_checked_on)
        {
            $final_arr = array();
            $url_cur = array();
            $url_past = array();
            $fails = array();
            $fails_past = array();

            $failed = $this->mongoDB->auditor->find(
                array('p_id' => $this->pid, 'a_id' => $this->week, 'tu' => 0,'src'=>1),
                array('_id' => 0, 'url' => 1)
            );
            if(!empty($pages)) {
                $failed->limit((int)$pages);
            }

            while ($ufs = $failed->getNext()) {
                $url_cur[] = trim(strip_tags(urldecode(urldecode(urldecode($ufs['url'])))));
            }

            $failed_p = $this->mongoDB->auditor->find(
                array('p_id' => $this->pid, 'a_id' => $this->past_week, 'tu' => 0),
                array('_id' => 0, 'url' => 1)
            );
            if(!empty($pages)) {
                $failed_p->limit((int)$pages);
            }
            while ($ufs2 = $failed_p->getNext()) {
                $url_past[] = trim(strip_tags(urldecode(urldecode(urldecode($ufs2['url'])))));
            }

            $url_new = array_diff($url_cur, $url_past);
            $url_deleted = array_diff($url_past, $url_cur);

            foreach ($url_new as $unvalue) {
                $final_arr['unique']['new'][] = $unvalue;
            }
            foreach ($url_deleted as $udvalue) {
                $final_arr['unique']['deleted'][] = $udvalue;
            }
            return $final_arr;
        }


        public function urlErrorPast($error, $pages)
        {
            $u_err = $this->mongoDB->auditor->find(
                array('p_id' => $this->pid, 'a_id' => $this->week, $error => array('$gt' => 0)),
                array('_id' => 0, 'url' => 1,'src'=>1)
            );
            if(!empty($pages)) {
                $u_err->limit((int)$pages);
            }
            while ($ufs = $u_err->getNext()) {
                $url_cur[] = trim(strip_tags(urldecode(urldecode(urldecode($ufs['url'])))));
            }

            $u_err2 = $this->mongoDB->auditor->find(
                array('p_id' => $this->pid, 'a_id' => $this->past_week, $error => array('$gt' => 0)),
                array('_id' => 0, 'url' => 1)
            );
            if(!empty($pages)) {
                $u_err2->limit((int)$pages);
            }
            while ($ufs2 = $u_err2->getNext()) {
                $url_past[] = trim(strip_tags(urldecode(urldecode(urldecode($ufs2['url'])))));
            }

            $url_new = array_diff($url_cur, $url_past);
            $url_deleted = array_diff($url_past, $url_cur);

            foreach ($url_new as $unvalue) {
                $final_arr['new'][] = $unvalue;
            }
            foreach ($url_deleted as $udvalue) {
                $final_arr['deleted'][] = $udvalue;
            }
            return $final_arr;
        }


        public function title_length()
        {
            $data = $this->mongoDB->auditor->findOne(
                array('p_id' => $this->pid, 'a_id' => $this->week, 'pos' => $this->pos),
                array('_id' => 0, 'etit.tit' => 1)
            );

            $title['Length'] = strlen($data['etit']['tit']);
            $title['Title'] = $data['etit']['tit'];

            return $title;
        }


        public function title_duplicate($count_only=null)
        {
            $data = $this->mongoDB->auditor->findOne(
                array('p_id' => $this->pid, 'a_id' => $this->week, 'pos' => $this->pos),
                array('_id' => 0, 'etit.tit' => 1)
            );

            if(!empty($count_only))
            {
                $dup_title_count = $this->mongoDB->auditor->count(
                    array('p_id' => $this->pid, 'a_id' => $this->week, 'etit.tit' => $data['etit']['tit'])
                );

                return $dup_title_count;
            }

            $title['Length'] = strlen($data['etit']['tit']);
            $title['Title'] = $data['etit']['tit'];

            $dup_title = $this->mongoDB->auditor->find(
                array('p_id' => $this->pid, 'a_id' => $this->week, 'etit.tit' => $data['etit']['tit']),
                array('_id' => 0, 'url' => 1)
            );
            while ($dups = $dup_title->getNext()) {
                if(!empty($dups['url'])) {
                    $title['url'][] = urldecode(urldecode(urldecode($dups['url'])));
                }
            }

            return $title;
        }


        public function meta_kw_length()
        {
            $data = $this->mongoDB->auditor->findOne(
                array('p_id' => $this->pid, 'a_id' => $this->week, 'pos' => $this->pos),
                array('_id' => 0, 'emet.kws' => 1)
            );

            $meta['Length'] = strlen($data['emet']['kws']);
            $meta['Keyword'] = $data['emet']['kws'];

            return $meta;
        }


        public function missing_alt()
        {
            $alts_urls = array();
            $missing_alt_arr = array();

            $miss_alt = $this->mongoDB->auditor->findOne(
                array('p_id' => $this->pid, 'a_id' => $this->week, 'pos'=>$this->pos, 'eimg.mal' => 1),
                array('_id' => 0, 'eimg.hrfmalt' => 1)
            );
            foreach ($miss_alt['eimg']['hrfmalt'] as $value) {
                if(!empty($value)) {
                    $alts_urls['Image HREFs Which Have Missing ALT Declaration'][] = $value['url'];
                }
            }

            return $alts_urls;
        }
        public function external_follow()
        {
            $extf_urls = array();

            $miss_alt = $this->mongoDB->auditor->findOne(
                array('p_id' => $this->pid, 'a_id' => $this->week, 'pos'=>$this->pos, 'eln.extfp' => 1),
                array('_id' => 0, 'eln.extf' => 1)
            );
            foreach ($miss_alt['eln']['extf'] as $value) {
                if(!empty($value)) {
                    $extf_urls[] = $value['link'];
                }
            }

            return $extf_urls;
        }
        public function external_nofollow()
        {
            $extnf_urls = array();

            $miss_alt = $this->mongoDB->auditor->findOne(
                array('p_id' => $this->pid, 'a_id' => $this->week, 'pos'=>$this->pos, 'eln.extnfp' => 1),
                array('_id' => 0, 'eln.extnf' => 1)
            );
            foreach ($miss_alt['eln']['extnf'] as $value) {
                if(!empty($value)) {
                    $extnf_urls[] = $value['link'];
                }
            }

            return $extnf_urls;
        }

        public function internal_follow()
        {
            $intf_urls = array();

            $miss_alt = $this->mongoDB->auditor->findOne(
                array('p_id' => $this->pid, 'a_id' => $this->week, 'pos'=>$this->pos, 'eln.intfp' => 1),
                array('_id' => 0, 'eln.intf' => 1)
            );
            foreach ($miss_alt['eln']['intf'] as $value) {
                if(!empty($value)) {
                    $intf_urls[] = $value['link'];
                }
            }

            return $intf_urls;
        }

        public function badres()
        {
            $bad_urls = array();

            $miss_alt = $this->mongoDB->auditor->findOne(
                array('p_id' => $this->pid, 'a_id' => $this->week, 'pos'=>$this->pos),
                array('_id' => 0, 'eln' => 1)
            );
            foreach ($miss_alt['eln']['badres'] as $key => $value) {
                $bad_urls[] = $value['link'];
            }
            return $bad_urls;
        }
        public function pagination_url()
        {
            $pagi_urls = array();

            $pagi_url = $this->mongoDB->auditor->findOne(
                array('p_id' => $this->pid, 'a_id' => $this->week, 'pos'=>$this->pos,'esht.pgu'=>1),
                array('_id' => 0, 'esht.pgurl' => 1)
            );
            foreach ($pagi_url['esht']['pgurl'] as $key => $value) {
                $pagi_urls[$key] = $value;
            }
            return $pagi_urls;
        }

        public function blocked_js()
        {
            $bjs_urls = array();

            $pagi_url = $this->mongoDB->auditor->findOne(
                array('p_id' => $this->pid, 'a_id' => $this->week, 'pos'=>$this->pos,'bjcs.bjs'=>1),
                array('_id' => 0, 'bjcs.blocked' => 1)
            );
            $bjs_urls =  $pagi_url['bjcs']['blocked']['js'];
            return $bjs_urls;
        }

        public function blocked_css()
        {
            $bcss_urls = array();

            $pagi_url = $this->mongoDB->auditor->findOne(
                array('p_id' => $this->pid, 'a_id' => $this->week, 'pos'=>$this->pos,'bjcs.bcss'=>1),
                array('_id' => 0, 'bjcs.blocked' => 1)
            );
            $bcss_urls =  $pagi_url['bjcs']['blocked']['css'];
            return $bcss_urls;
        }

        public function meta_kw_duplicate($count_only=null)
        {
            $data = $this->mongoDB->auditor->findOne(
                array('p_id' => $this->pid, 'a_id' => $this->week, 'pos' => $this->pos),
                array('_id' => 0, 'emet.kws' => 1)
            );

            if(!empty($count_only))
            {
                $dup_meta_kw_count = $this->mongoDB->auditor->count(
                    array('p_id' => $this->pid, 'a_id' => $this->week, 'emet.kws' => $data['emet']['kws'])
                );

                return $dup_meta_kw_count;
            }

            $meta['Length'] = strlen($data['emet']['kws']);
            $meta['Keyword'] = $data['emet']['kws'];

            $dup_meta_kw = $this->mongoDB->auditor->find(
                array('p_id' => $this->pid, 'a_id' => $this->week, 'emet.kws' => $data['emet']['kws']),
                array('_id' => 0, 'url' => 1)
            );
            while ($dups = $dup_meta_kw->getNext()) {
                $meta['url'][] = urldecode(urldecode(urldecode($dups['url'])));
            }

            return $meta;
        }

        public function meta_desc_length()
        {
            $data = $this->mongoDB->auditor->findOne(
                array('p_id' => $this->pid, 'a_id' => $this->week, 'pos' => $this->pos),
                array('_id' => 0, 'emet.desc' => 1)
            );

            $meta['Length'] = strlen($data['emet']['desc']);
            $meta['Description'] = $data['emet']['desc'];

            return $meta;
        }


        public function meta_desc_duplicate($count_only=null)
        {
            $data = $this->mongoDB->auditor->findOne(
                array('p_id' => $this->pid, 'a_id' => $this->week, 'pos' => $this->pos),
                array('_id' => 0, 'emet.desc' => 1)
            );

            if(!empty($count_only))
            {
                $dup_meta_desc_count = $this->mongoDB->auditor->count(
                    array('p_id' => $this->pid, 'a_id' => $this->week, 'emet.desc' => $data['emet']['desc'])
                );

                return $dup_meta_desc_count;
            }

            $meta['Length'] = strlen($data['emet']['desc']);
            $meta['Description'] = $data['emet']['desc'];

            $dup_meta_desc = $this->mongoDB->auditor->find(
                array('p_id' => $this->pid, 'a_id' => $this->week, 'emet.desc' => $data['emet']['desc']),
                array('_id' => 0, 'url' => 1)
            );
            while ($dups = $dup_meta_desc->getNext()) {
                $meta['url'][] = urldecode(urldecode(urldecode($dups['url'])));
            }

            return $meta;
        }


        public function url_length()
        {
            $data = $this->mongoDB->auditor->findOne(
                array('p_id' => $this->pid, 'a_id' => $this->week, 'pos' => $this->pos),
                array('_id' => 0, 'url' => 1)
            );

            $url['Length'] = strlen($data['url']);
            $url['url'] = urldecode(urldecode(urldecode($data['url'])));

            return $url;
        }


        public function links_int()
        {
            $data = $this->mongoDB->auditor->findOne(
                array('p_id' => $this->pid, 'a_id' => $this->week, 'pos' => $this->pos),
                array('_id' => 0, "eln.intnf" => 1, "eln.intf" => 1, "eln.internal" => 1,"eln.int" =>1)
            );
            if(!empty($data['eln']['intf'])) {
                foreach ($data['eln']['intf'] as $value) {
                    $value = urldecode(urldecode(urldecode($value['link'])));
                    $f_arr[] = $value;
                }
            }

            if(!empty($data['eln']['intnf'])) {
                foreach ($data['eln']['intnf'] as $nf_value) {
                    $nf_value = urldecode(urldecode(urldecode($nf_value['link'])));
                    $nf_arr[] = $nf_value;
                }
            }

            if(!empty($data['eln']['internal'])) {
                foreach ($data['eln']['internal'] as $nf_value) {
                    $ln_value = urldecode(urldecode(urldecode($nf_value['link'])));
                    $ln_arr[] = $ln_value;
                }
            }


            if(!empty($data['eln']['int'])) $links['Count'] = $data['eln']['int'];
            if(!empty($f_arr)) $links['Follow'] = $f_arr;
            if(!empty($nf_arr)) $links['No Follow'] = $nf_arr;
            if(!empty($ln_arr)) $links['internal'] = $ln_arr;
            return $links;
        }

        public function links_ext()
        {
            $data = $this->mongoDB->auditor->findOne(
                array('p_id' => $this->pid, 'a_id' => $this->week, 'pos' => $this->pos),
                array('_id' => 0, "eln.extnf" => 1, "eln.extf" => 1, "eln.external" => 1,"eln.ext" =>1)
            );
            
            if(!empty($data['eln']['extf'])) {
                foreach ($data['eln']['extf'] as $value) {
                    $value = urldecode(urldecode(urldecode($value['link'])));
                    $f_arr[] = $value;
                }
            }


            if(!empty($data['eln']['extnf'])) {
                foreach ($data['eln']['extnf'] as $nf_value) {
                    $nf_value = urldecode(urldecode(urldecode($nf_value['link'])));
                    $nf_arr[] = $nf_value;
                }
            }

            if(!empty($data['eln']['external'])) {
                foreach ($data['eln']['external'] as $nf_value) {
                    $ln_value = urldecode(urldecode(urldecode($nf_value['link'])));
                    $ln_arr[] = $ln_value;
                }
            }

            if(!empty($data['eln']['ext'])) $links['Count'] = $data['eln']['ext'];
            if(!empty($data['eln']['extf'])) $links['follow'] = $f_arr;
            if(!empty($data['eln']['extnf'])) $links['nofollow'] = $nf_arr;
            if(!empty($ln_arr)) $links['external'] = $ln_arr;
           
            return $links;
        }


        public function links_int_nofollow()
        {
            $data = $this->mongoDB->auditor->findOne(
                array('p_id' => $this->pid, 'a_id' => $this->week, 'pos' => $this->pos),
                array('_id' => 0, "eln.nf.int" => 1)
            );

            $links['Length'] = count($data['eln']['nf']['int']);
            $links['Links'] = $data['eln']['nf']['int'];

            return $links;
        }

        public function links_ext_nofollow()
        {
            $data = $this->mongoDB->auditor->findOne(
                array('p_id' => $this->pid, 'a_id' => $this->week, 'pos' => $this->pos),
                array('_id' => 0, "eln.nf.ext" => 1)
            );

            $links['Length'] = count($data['eln']['nf']['ext']);
            $links['Links'] = $data['eln']['nf']['ext'];

            return $links;
        }

        public function links_int_follow()
        {
            $data = $this->mongoDB->auditor->findOne(
                array('p_id' => $this->pid, 'a_id' => $this->week, 'pos' => $this->pos),
                array('_id' => 0, "eln.f.int" => 1)
            );

            $links['Length'] = count($data['eln']['f']['int']);
            $links['Links'] = $data['eln']['f']['int'];

            return $links;
        }

        public function links_ext_follow()
        {
            $data = $this->mongoDB->auditor->findOne(
                array('p_id' => $this->pid, 'a_id' => $this->week, 'pos' => $this->pos),
                array('_id' => 0, "eln.f.ext" => 1)
            );

            $links['Length'] = count($data['eln']['f']['ext']);

            $links['Links'] = $data['eln']['f']['ext'];

            return $links;
        }


        public function content_duplicate()
        {
            $final_arr = array();

            $dup_content = $this->mongoDB->auditor->findOne(
                array('p_id' => $this->pid, 'a_id' => $this->week, 'tu' => array('$gt' => 0), 'pos'=>$this->pos),
                array('_id' => 0, 'url' => 1, 'dw' => 1)
            );

            $all_urls = $this->mongoDB->auditor->find(
                array('p_id' => $this->pid, 'a_id' => $this->week),
                array('_id' => 0, 'url' => 1, 'pos' => 1)
            );
            while ($urls = $all_urls->getNext()) {
                $urls_arr[$urls['pos']] = urldecode(urldecode(urldecode($urls['url'])));
            }

            $dup_con = array();
            $dup_con['url'] = urldecode(urldecode(urldecode($dup_content['url'])));
            foreach ($dup_content['dw'] as $key => $value) {
                $final_arr[$dup_con['url']]['Duplicates'][$urls_arr[$value['pos']]] = $value['perc'];
            }
                return $final_arr;
        }

        public function img_many()
        {
            $many_img = $this->mongoDB->auditor->findOne(
                array('p_id' => $this->pid, 'a_id' => $this->week, 'pos' => $this->pos),
                array('_id' => 0, 'url' => 1, 'eimg.t'=> 1,'eimg.hrfmalt'=>1,'eimg.hrfalt'=>1)
            );
            $final_arr['Count'] = $many_img['eimg']['t'];
            foreach ($many_img['eimg']["hrfalt"] as $e_href) {
                $final_arr['Image HREFs'][] = $e_href['url'];
            }
            foreach ($many_img['eimg']["hrfmalt"] as $e_href) {
                $final_arr['Image HREFs'][] = $e_href['url'];
            }
            return $final_arr;
        }

        public function sheets_many()
        {
            $many_img = $this->mongoDB->auditor->findOne(
                array('p_id' => $this->pid, 'a_id' => $this->week, 'pos' => $this->pos),
                array('_id' => 0, 'url' => 1, 'esht.sts' => 1)
            );

            $final_arr['Count'] = count($many_img['esht']['sts']);
            $final_arr['Styles Sheets'] = $many_img['esht']['sts'];
            return $final_arr;
        }

        public function spell_mistakes()
        {
            $many_img = $this->mongoDB->auditor->findOne(
                array('p_id' => $this->pid, 'a_id' => $this->week, 'pos' => $this->pos),
                array('_id' => 0, 'url' => 1, 'espl.mis' => 1)
            );

            $final_arr['Count'] = count($many_img['espl']['mis']);
            $final_arr['Mistakes'] = $many_img['espl']['mis'];
            return $final_arr;
        }

        public function many_h1()
        {
            $many_h1 = $this->mongoDB->auditor->findOne(
                array('p_id' => $this->pid, 'a_id' => $this->week, 'pos' => $this->pos),
                array('_id' => 0, 'url' => 1, 'eh1s.h1t' => 1)
            );

            $final_arr['H1 Tags'] = $many_h1['eh1s']['h1t'];
            return $final_arr;
        }

        public function many_h2()
        {
            $many_h2 = $this->mongoDB->auditor->findOne(
                array('p_id' => $this->pid, 'a_id' => $this->week, 'pos' => $this->pos),
                array('_id' => 0, 'url' => 1, 'eh2s.h2t' => 1)
            );

            $final_arr['H2 Tags'] = $many_h2['eh2s']['h2t'];
            return $final_arr;
        }

        public function open_graph()
        {
            $final_arr = array();
            $open_graphs = $this->mongoDB->auditor->findOne(
                array('p_id' => $this->pid, 'a_id' => $this->week, 'pos' => $this->pos),
                array('_id' => 0, 'url' => 1, 'twt' => 1)
            );
            foreach ($open_graphs['twt']['og'] as $key => $value) {
                $final_arr[][$key] = $value;
            }
            return $final_arr;
        }
        public function twitter_card()
        {
            $final_arr = array();
            $twitter_card = $this->mongoDB->auditor->findOne(
                array('p_id' => $this->pid, 'a_id' => $this->week, 'pos' => $this->pos),
                array('_id' => 0, 'url' => 1, 'twt' => 1)
            );
            foreach ($twitter_card['twt']['twitter'] as $key => $value) {
                $final_arr[][$key] = $value;
            }
            return $final_arr;
        }
    }

?>