<?php

require_once __DIR__.'/Common.php';

class Services
{
    private $common;
    private $con_dist;
    private $dbc;
    private $temp_db;
    private $temp_user;
    private $temp_pass;

    public function __construct()
    {
        $this->common = new Common();
        $this->dbc = new Database();

        $dist = $this->dbc->conDist();
        $this->temp_pass = "yb9738z";
        $this->temp_user = "auditor";
        $this->temp_db = "auditor_temp";

        if ($dist instanceof mysqli) {
            $this->con_dist = $dist;
        } else {
            $this->common->logErrorFatal('', 'mysql:', 'dist mysql connection object is unusable'.print_r($dist, true), print_r(error_get_last(), true));
        }
    }


    public function chkProjExists($pid)
    {
        $findProj = "SELECT p_id,domain,user_id FROM sa_projects WHERE p_hash = '".$pid."'";

       if ($findP = mysqli_query($this->con_dist, $findProj)) {
            if(mysqli_num_rows($findP)) {
                $data = mysqli_fetch_assoc($findP);
                return $data;
            } else {
                return false;
            }
        } else {
            $this->common->logErrorFatal('', 'mysql:', mysqli_error($this->con_dist), print_r(error_get_last(), true));

            $result = array(
                'response' => 'Internal Error Occured',
                'response_code' => 101
            );

            echo json_encode($result);
            exit();
        }
    }


    public function calculatePriority($u_class, $p_class)
    {
        if($u_class == 1 && $p_class == 1) {
            $priority = 1;
        }
        elseif($u_class == 2 && $p_class == 1) {
            $priority = 2;
        }
        elseif($u_class == 2 && $p_class == 2) {
            $priority = 3;
        }
        elseif($u_class == 3 && $p_class == 2) {
            $priority = 4;
        }
        elseif($p_class == 3) {
            $priority = 5;
        }
        else {
            $priority = 5;
            // $this->common->logErrorFatal('', "service:incorrect priority params", "priority:u_class:$u_class,p_class:$p_class", print_r(error_get_last(), true));

            // $result = array(
            //     'response' => 'Incorect priority combination',
            //     'response_code' => 101
            // );

            // echo json_encode($result);
            // exit();
        }
        return $priority;
    }


    // public function getUserQuota($uid)
    // {
    //     $quota = exec("curl http://rankwatch.com/getuserquota?$uid; echo");

    //     if(empty($quota)) {
    //         $this->common->logErrorFatal('', "service:", "unable to get quota: $uid", print_r(error_get_last(), true));

    //         $result = array(
    //             'response' => 'Unable to retrive user quota',
    //             'response_code' => 101
    //         );

    //         echo json_encode($result);
    //         exit();
    //     } else {
    //         return $quota;
    //     }
    // }


    // public function updateUserQuota($uid, $new_quota)
    // {

    // }

    // public function getProjPages($pid)
    // {
    //     $qry = "SELECT pages_to_crawl FROM projects WHERE id_hash='".$pid."'";

    //     if($chk = mysqli_query($this->con_dist, $qry)) {
    //         if(mysqli_num_rows($chk)) {
    //             $res = mysqli_fetch_assoc($chk);
    //             if(empty($res['pages_to_crawl']) && $res['pages_to_crawl'] !== 0) {
    //                 $this->common->logErrorFatal('', "service:", "mysql:$qry", print_r(error_get_last(), true));
    //                 $msg = "Unable to get PID pages";
    //                 $err = 1;
    //             } else {
    //                 return $res['pages_to_crawl'];
    //             }
    //         } else {
    //             $this->common->logErrorFatal('', "service:", "no entry found for $pid", print_r(error_get_last(), true));
    //             $msg = "No entry found for PID";
    //             $err = 1;
    //         }
    //     } else {
    //         $this->common->logErrorFatal('', "service:", mysqli_error($this->con_dist), print_r(error_get_last(), true));
    //         $msg = "Internal Error";
    //         $err = 1;
    //     }

    //     if(!empty($err)) {
    //         $result = array(
    //             'response' => $msg,
    //             'response_code' => 101
    //         );

    //         echo json_encode($result);
    //         exit();
    //     }
    // }


    public function getUserProjCount($uid, $pid)
    {
        $qry = "SELECT COUNT(p_id) AS total_pages FROM sa_projects WHERE user_id=$uid AND p_id != $pid AND expired=0";

        if($chk = mysqli_query($this->con_dist, $qry)) {
            if(mysqli_num_rows($chk)) {
                $res = mysqli_fetch_assoc($chk);
                if(empty($res['total_pages']) && $res['total_pages'] != 0) {
                    $this->common->logErrorFatal('', "service:", "mysql:$qry", print_r(error_get_last(), true));
                    $msg = "Unable to get PID pages";
                    $err = 1;
                } else {
                    return $res['total_pages'];
                }
            } else {
                $this->common->logErrorFatal('', "service:", "no entry found for $uid", print_r(error_get_last(), true));
                $msg = "No entry found for PID";
                $err = 1;
            }
        } else {
            $this->common->logErrorFatal('', "service:", mysqli_error($this->con_dist), print_r(error_get_last(), true));
            $msg = "Internal Error";
            $err = 1;
        }

        if(!empty($err)) {
            $result = array(
                'response' => $msg,
                'response_code' => 101
            );

            echo json_encode($result);
            exit();
        }
    }


    public function getAllProjPages($uid, $pid, $new="")
    {
        if(!empty($new)) {
            $qry = "SELECT SUM(pages_to_crawl) AS total_pages FROM sa_projects WHERE user_id=$uid AND expired=0";
        } else {
            $qry = "SELECT SUM(pages_to_crawl) AS total_pages FROM sa_projects WHERE user_id=$uid AND p_id != $pid AND expired=0";
        }

        if($chk = mysqli_query($this->con_dist, $qry)) {
            if(mysqli_num_rows($chk)) {
                $res = mysqli_fetch_assoc($chk);
                if(empty($res['total_pages']) && $res['total_pages'] != 0) {
                    $this->common->logErrorFatal('', "service:", "mysql:$qry", print_r(error_get_last(), true));
                    $msg = "Unable to get PID pages";
                    $err = 1;
                } else {
                    return $res['total_pages'];
                }
            } else {
                $this->common->logErrorFatal('', "service:", "no entry found for $uid", print_r(error_get_last(), true));
                $msg = "No entry found for PID";
                $err = 1;
            }
        } else {
            $this->common->logErrorFatal('', "service:", mysqli_error($this->con_dist), print_r(error_get_last(), true));
            $msg = "Internal Error";
            $err = 1;
        }

        if(!empty($err)) {
            $result = array(
                'response' => $msg,
                'response_code' => 101
            );

            echo json_encode($result);
            exit();
        }
    }

    // public function chkQuotaLimit($u_id, $p_id, $new_pages, $p_class) {
    //     if($p_class == 1) {
    //         $add_pages = $new_pages;
    //     }
    //     elseif($p_class == 2) {
    //         $old_pages = $this->getProjPages($p_id);
    //         $add_pages = $new_pages - $old_pages;
    //     }

    //     $total_old_pages = $this->getAllProjPages($u_id);

    //     $new_quota = $total_old_pages + $add_pages;
    //     $current_quota = $this->getUserQuota($u_id);

    //     if($new_quota > $current_quota) {
    //         $result = array(
    //             'response' => "Project pages exceed Quota limit",
    //             'response_code' => 101
    //         );

    //         echo json_encode($result);
    //         exit();
    //     }
    //     elseif($new_quota <= $current_quota) {
    //         return true;
    //     }

    // }


    public function getPerUrlGraph($data)
    {
        $url = $data['url'];
        $graph_data = array();
        $desc_arr = array(
            'mal' => array('Missing Image Alt', 'have missing ALT in images', 'eimg.mal'),
            'mkw' => array('Missing Meta Keyword', 'have meta keywords missing', 'emet.mkw'),
            'lkw' => array('Long Meta Keyword', 'have too long meta keyword', 'emet.lkw'),
            'skw' => array('Short Meta Keyword', 'have too short meta keyword', 'emet.skw'),
            'mds' => array('Missing Meta Description', 'have meta description missing', 'emet.mds'),
            'lds' => array('Long Meta Description', 'have too long meta description', 'emet.lds'),
            'sds' => array('Short Meta Description', 'have too short meta keyword', 'emet.sds'),
            'thn' => array('Thin Content', 'have thin content', 'epage.thn'),
            'low' => array('Text to Html Ratio', 'have low text to html ratio', 'etth.low'),
            'dti' => array('Duplicate Title', 'have duplicate title in them', 'etit.dti'),
            'dkw' => array('Duplicate Meta Keywords', 'have duplicate meta keywords in them', 'emet.dkw'),
            'dde' => array('Duplicate Meta Description', 'have duplicate meta description in them', 'emet.dde'),
            'tu' => array('Duplicate Content', 'have duplicate content in them', 'tu'),
            'lt' => array('Long Title', 'have too long title', 'etit.lt'),
            'sti' => array('Short Title', 'have too short title', 'etit.sti'),
            'eti' => array('Empty/Missing Title', 'have empty title', 'etit.eti'),
            'mrf' => array('Meta Refresh', 'have meta refresh', 'emet.mrf'),
            'mre' => array('Meta Redirect', 'have meta redirect', 'emet.mre'),
            'mnf' => array('Meta Nofollow', 'have meta nofollow', 'emet.mnf'),
            'mlg' => array('Missing Language Declaration', 'have language declaration missing', 'mlg'),
            'mst' => array('Too Many Stylesheets', 'have too many stylesheets included', 'esht.mst'),
            'mh1' => array('Missing H1 Tag', 'have missing h1 tag', 'eh1s.mh1'),
            'eh1' => array('Too Many H1 Tags', 'have too many h1 tags', 'eh1s.eh1'),
            'mh2' => array('Missing H2 Tag', 'have missing h2 tag', 'eh2s.mh2'),
            'eh2' => array('Too Many H2 Tags', 'have too many h2 tags', 'eh2s.eh2'),
            'lu' => array('Long URL', 'have too long URL', 'eurl.lu'),
            'spu' => array('Special Characters in URL', 'have special characters in URL', 'eurl.spu'),
            'unu' => array('Underscore Characters in URL', 'have underscores in URL', 'eurl.unu'),
            'upu' => array('UpperCase Characters in URL', 'have uppercase characters in URL', 'eurl.upu'),
            'frm' => array('Using Iframes', 'have iframes in content', 'efrm.frm'),
            'fls' => array('Using Flash', 'have flash in content', 'efls.fls'),
            'ent' => array('Too Many Internal Links', 'have too many internal links', 'eln.ent'),
            'eex' => array('Too Many External Links', 'have too many external links', 'eln.eex'),
            'nfc' => array('NoFollow Links', 'have nofollow links', 'eln.nfc'),
            'eim' => array('Too many images', 'have too many images', 'eimg.eim'),
            // 'esp' => array('Too many spelling mistakes', 'have too many spelling mistakes', 'espl.esp'),
            't' => array('Slow Page Speed', 'have slow response time', 't'),
            'big' => array('Page Size too Big', 'have big page size', 'epage.big'),
            'cms' => array('Missing Characterset Declaration', 'have characterset declaration missing', 'emet.cms'),
            'dms' => array('Missing Doctype Declaration', 'have doctype declaration missing', 'edoc.dms'),
            'bjs' => array('Blocked JS Files', 'have blocked JS', 'bjcs.bjs'),
            'bcss' => array('Blocked CSS Files', 'have blocked CSS', 'bjcs.bcss'),
            'ehrfl' => array('Href Lang error', 'have Href Lang Error', 'ehrfl'),
            'emob' => array('Mobile Friendly error', 'have mobile friendly issue', 'mob_data.emob'),
            'ehid'=>array('Hidden Content on page', 'have hidden content on page', 'ehid')
        );
        foreach ($desc_arr as $eckey => $ecvalue) {
            $err_col_arr[$ecvalue[2]] = 1;
        }


        $qry = "SELECT p_id,domain,duplicated FROM sa_projects WHERE p_hash='".$data['p_id']."'";
        if($chk = mysqli_query($this->con_dist, $qry)) {
            if(mysqli_num_rows($chk)) {
                $res = mysqli_fetch_assoc($chk);
                $pid = $res['p_id'];

                if($res['duplicated'] < 2) {
                    $qry = "SELECT p_id FROM sa_projects WHERE domain='".$res['domain']."' AND duplicated<2";
                    $chk = mysqli_query($this->con_dist, $qry);
                    $res = mysqli_fetch_assoc($chk);
                    $pid = (int)$res['p_id'];
                }

            }
        } else {
            $this->common->logErrorFatal('', "mysql:$qry", mysqli_error($this->con_dist), print_r(error_get_last(), true));
        }

        $mongo = $this->dbc->conMongo();
        if (!is_object($mongo)) {
            $this->common->logErrorFatal('', 'mongo:', 'mongo connection object is unusable'.print_r($mongo, true), print_r(error_get_last(), true));
            return;
        }

        $u_his = $mongo->auditor->find(
            array('p_id' => (int)$pid, 'url' => $url),
            $err_col_arr
        );
        $u_his->limit(4);

        while($url_errs = $u_his->getNext()) {
            $e_err_arr[] = $url_errs;
        }


        foreach ($e_err_arr as $erval) {
            $created_on = $erval['_id']->getTimestamp();
            $created_on = date('Y-m-d', $created_on);
            unset($erval['_id']);

            if(!array_key_exists($created_on, $date_arr)) {
                $date_arr[$created_on] = 1;
                $each_url_arr = array();

                foreach ($erval as $eskey => $esvalue) {
                    if(is_array($esvalue))
                    {
                        foreach ($esvalue as $esk => $esv) {
                            if(!empty($esvalue[$esk]) && $esvalue[$esk] >= 1) {
                                $page_errors[$esk] = 1;
                            }
                        }
                    }
                    elseif(!empty($erval[$eskey]) && $erval[$eskey] >= 1) {
                        $page_errors[$eskey] = 1;
                    }
                }

                if($erval['t'] > T_SPEED_SLOW)
                {
                    if(empty($page_errors['t']))
                        $page_errors['t'] = 0;

                    $page_errors['t'] += 1;
                }
                $graph_data[] = $page_errors;
                $graph_date[] = date('Y-m-d', $created_on);
            }
        }

        $result = array(
            'response' => "setting data",
            'data' => $graph_data,
            'date' => $graph_date,
            'response_code' => 100
        );
        return json_encode($result);

    }


    public function getPerErrorGraph($data)
    {

        $error = $data['error'];
        $show = $data['show'];
        $graph_data = array();
        $graph_date = array();
        $desc_arr = array(
            'mal' => array('Missing Image Alt', 'have missing ALT in images', 'eimg.mal'),
            'mkw' => array('Missing Meta Keyword', 'have meta keywords missing', 'emet.mkw'),
            'lkw' => array('Long Meta Keyword', 'have too long meta keyword', 'emet.lkw'),
            'skw' => array('Short Meta Keyword', 'have too short meta keyword', 'emet.skw'),
            'mds' => array('Missing Meta Description', 'have meta description missing', 'emet.mds'),
            'lds' => array('Long Meta Description', 'have too long meta description', 'emet.lds'),
            'sds' => array('Short Meta Description', 'have too short meta keyword', 'emet.sds'),
            'thn' => array('Thin Content', 'have thin content', 'epage.thn'),
            'low' => array('Text to Html Ratio', 'have low text to html ratio', 'etth.low'),
            'dti' => array('Duplicate Title', 'have duplicate title in them', 'etit.dti'),
            'dkw' => array('Duplicate Meta Keywords', 'have duplicate meta keywords in them', 'emet.dkw'),
            'dde' => array('Duplicate Meta Description', 'have duplicate meta description in them', 'emet.dde'),
            'tu' => array('Duplicate Content', 'have duplicate content in them', 'tu'),
            'lt' => array('Long Title', 'have too long title', 'etit.lt'),
            'sti' => array('Short Title', 'have too short title', 'etit.sti'),
            'eti' => array('Empty/Missing Title', 'have empty title', 'etit.eti'),
            'mrf' => array('Meta Refresh', 'have meta refresh', 'emet.mrf'),
            'mre' => array('Meta Redirect', 'have meta redirect', 'emet.mre'),
            'mnf' => array('Meta Nofollow', 'have meta nofollow', 'emet.mnf'),
            'mlg' => array('Missing Language Declaration', 'have language declaration missing', 'mlg'),
            'mst' => array('Too Many Stylesheets', 'have too many stylesheets included', 'esht.mst'),
            'mh1' => array('Missing H1 Tag', 'have missing h1 tag', 'eh1s.mh1'),
            'eh1' => array('Too Many H1 Tags', 'have too many h1 tags', 'eh1s.eh1'),
            'mh2' => array('Missing H2 Tag', 'have missing h2 tag', 'eh2s.mh2'),
            'eh2' => array('Too Many H2 Tags', 'have too many h2 tags', 'eh2s.eh2'),
            'lu' => array('Long URL', 'have too long URL', 'eurl.lu'),
            'spu' => array('Special Characters in URL', 'have special characters in URL', 'eurl.spu'),
            'unu' => array('Underscore Characters in URL', 'have underscores in URL', 'eurl.unu'),
            'upu' => array('UpperCase Characters in URL', 'have uppercase characters in URL', 'eurl.upu'),
            'frm' => array('Using Iframes', 'have iframes in content', 'efrm.frm'),
            'fls' => array('Using Flash', 'have flash in content', 'efls.fls'),
            'ent' => array('Too Many Internal Links', 'have too many internal links', 'eln.ent'),
            'eex' => array('Too Many External Links', 'have too many external links', 'eln.eex'),
            'nfc' => array('NoFollow Links', 'have nofollow links', 'eln.nfc'),
            'eim' => array('Too many images', 'have too many images', 'eimg.eim'),
            // 'esp' => array('Too many spelling mistakes', 'have too many spelling mistakes', 'espl.esp'),
            't' => array('Slow Page Speed', 'have slow response time', 't'),
            'big' => array('Page Size too Big', 'have big page size', 'epage.big'),
            'cms' => array('Missing Characterset Declaration', 'have characterset declaration missing', 'emet.cms'),
            'dms' => array('Missing Doctype Declaration', 'have doctype declaration missing', 'edoc.dms'),
            'bjs' => array('Blocked JS Files', 'have blocked JS', 'bjcs.bjs'),
            'bcss' => array('Blocked CSS Files', 'have blocked CSS', 'bjcs.bcss'),
            'ehrfl' => array('Href Lang error', 'have Href Lang Error', 'ehrfl'),
            'emob' => array('Mobile Friendly error', 'have mobile friendly issue', 'mob_data.emob'),
            'ehid'=>array('Hidden Content on page', 'have hidden content on page', 'ehid')
        );


        $qry = "SELECT p_id,domain,duplicated FROM sa_projects WHERE p_hash='".$data['p_id']."'";
        if($chk = mysqli_query($this->con_dist, $qry)) {
            if(mysqli_num_rows($chk)) {
                $res = mysqli_fetch_assoc($chk);
                $pid = $res['p_id'];

                if($res['duplicated'] < 2) {
                    $qry = "SELECT p_id FROM sa_projects WHERE domain='".$res['domain']."' AND duplicated<2";
                    $chk = mysqli_query($this->con_dist, $qry);
                    $res = mysqli_fetch_assoc($chk);
                    $pid = (int)$res['p_id'];
                }

            }
        } else {
            $this->common->logErrorFatal('', "mysql:$qry", mysqli_error($this->con_dist), print_r(error_get_last(), true));
        }

        $mongo = $this->dbc->conMongo();
        if (!is_object($mongo)) {
            $this->common->logErrorFatal('', 'mongo:', 'mongo connection object is unusable'.print_r($mongo, true), print_r(error_get_last(), true));
            return;
        }

        $qry2 = "SELECT created_on,auditor_id FROM history WHERE p_id=$pid ORDER BY auditor_id ASC LIMIT 10";
        if($chk2 = mysqli_query($this->con_dist, $qry2)) {
            if(mysqli_num_rows($chk2)) {
                while($res2 = mysqli_fetch_assoc($chk2)) {
                    if ($error == 'ehrfl')
                    {
                        $u_err_c = $mongo->hreflang_data->count(
                            array('p_id'=> (int)$pid, 'a_id'=> (int)$res2['auditor_id'],'err_status'=>1)
                        );
                    }
                    else{
                        $u_err_c = $mongo->auditor->count(
                            array('p_id' => (int)$pid, 'a_id' => (int)$res2['auditor_id'], $error => array('$gt' => 0))
                        );
                    }

                    $w_date = date('Y-m-d', strtotime($res2['created_on']));
                    $graph_data[] = $u_err_c;
                    $graph_date[] = $w_date;
                }
            }
        } else {
            $this->common->logErrorFatal('', "mysql:$qry2", mysqli_error($this->con_dist), print_r(error_get_last(), true));
        }

        $result = array(
            'response' => "graph data",
            'data' => $graph_data,
            'date' => $graph_date,
            'err_name' => $desc_arr[$show][0],
            'response_code' => 100
        );
        return json_encode($result);
    }


    /**
     * Service to create runnow in Website Auditor
     *
     * Accepts GET params only
     *
     * @param int user_id
     * @param int pages_to_crawl
     * @param string p_id
     * @param string project (domain_name)
     * @param int user_class (1=>new free, 2=>paid, 3=>free)
     * @param int project_class (1=>new, 2=>quota_increase, 3=>weekly)
     *
     * Conditions:
     *   Create Entry:
     *       if no entry for PID
     *
     *   Update Entry:
     *       if entry exists for PID
     *
     *   Errors:
     *       if unable to create or update entry
     *       if GET parameters empty
     *       if PID exists but is associated with other domain
     *       if PID exists but is associated with other user
     *       if PID exists and is currently processing
     *
     *   Success:
     *       if entry created/updated successfully
     *
     *   Return Value:
     *       JSON encoded array,
     *       VALUES:
     *            response: response message
     *            response_code: 100(success)/101(error)
     *
     */
    public function newProjRunnow($data)
    {
        $resp_code = 101;

        if(!isset($data['user_id']) || !isset($data['pages_to_crawl']) || !isset($data['p_id']) || !isset($data['project'])|| !isset($data['user_class'])|| !isset($data['project_class'])) {

            $this->common->logErrorFatal('', 'service:', print_r($data, true), print_r(error_get_last(), true));

            $result = array(
                'response' => 'Incomplete Data',
                'response_code' => $resp_code
            );

            return json_encode($result);
        }

        $user_id = $data['user_id'];
        $pages_to_crawl = (int)$data['pages_to_crawl'];
        $p_id = $data['p_id'];
        $pid = (int)$data['pid'];
        $domain = $data['project'];
        $p_class = $data['project_class'];
        $u_class = $data['user_class'];
        $user_quota = $data['user_quota'];
        $source = $data['source'];
       
        
        $pages_to_crawl = 0;
        if ($source != 'dev'){
            $result = array(
                'response' => "NO Permission to add project",
                'response_code' => 109
            );
            return json_encode($result);
        }

        //$makeProj = "INSERT INTO sa_projects(p_hash,user_id,domain,pages_to_crawl,runnow,expired,cstatus,duplicated,history_created,crawl_no) VALUES('".$p_id."',$user_id,'".$domain."',$pages_to_crawl,0,0,'0',0,0,0)";
        
        $makeProj = "INSERT INTO sa_projects(p_id,p_hash,user_id,domain,created) VALUES($pid, '".$p_id."',$user_id,'".$domain."', ".time().")";


        /*
        $pid = $this->getIdFromHash($data['p_id']);

        $tot_used = $this->getAllProjPages($data['user_id'], $pid, 1);
        $priority =  $this->calculatePriority($u_class, $p_class);

        */
        // if(($tot_used + $pages_to_crawl) > $user_quota && $tot_used < $user_quota) {
        //     $pages_to_crawl = $user_quota - $tot_used;
        //     $makeProj = "INSERT INTO projects(id_hash,user_id,domain,pages_to_crawl,runnow,runnow_priority) VALUES('".$p_id."',$user_id,'".$domain."',$pages_to_crawl,1,$priority)";
        // }
        /*
        if(($tot_used + $pages_to_crawl) <= $user_quota) {
            $pages_to_crawl = (int)$data['pages_to_crawl'];
            $makeProj = "INSERT INTO sa_projects(p_hash,user_id,domain,pages_to_crawl,runnow,crawl_no) VALUES('".$p_id."',$user_id,'".$domain."',$pages_to_crawl,0,1)";
        } else {
            $pages_to_crawl = 0;
            $makeProj = "INSERT INTO sa_projects(p_hash,user_id,domain,pages_to_crawl,runnow,expired,cstatus,duplicated,history_created,crawl_no) VALUES('".$p_id."',$user_id,'".$domain."',$pages_to_crawl,0,2,8,2,2,1";
        }
        */
        // check if entry already exists for this project
        if($existingPID = $this->chkProjExists($p_id)) {
            $msg = 'PID already exists, runnow can\'t be created for existing PID';
        } else {

            if (mysqli_query($this->con_dist, $makeProj)) {
                // $this->updateUserQuota($user_id, $new_quota);
                $msg = 'Runnow created successfully';
                $resp_code = 100;
            }
            else {
                var_dump(mysqli_error($this->con_dist));die;
                $this->common->logErrorFatal('', 'mysql:', mysqli_error($this->con_dist), print_r(error_get_last(), true));
                $msg = 'Unable to create runnow';
            }
        }
        $result = array(
            'response' => $msg,
            'response_code' => $resp_code
        );
        return json_encode($result);
    }


    public function delProject($data)
    {
        $qry = "SELECT * FROM sa_projects WHERE p_hash='".$data['p_id']."'";
        if($chk = mysqli_query($this->con_dist, $qry)) {
            $res = mysqli_fetch_assoc($chk);
            // if($res['crawled'] != 2 || $res['scraped'] != 2 || $res['processed'] != 2 || $res['duplicated'] != 2 || $res['history_created'] != 2) {
            //     $result = array(
            //         'response' => "Project is currently processing, please wait until it's finished.",
            //         'response_code' => 101
            //     );
            //     return json_encode($result);
            // }
            $pid = $res['p_id'];

            if(!empty($pid)) {
                $del = "DELETE FROM sa_projects WHERE p_id=$pid";
                if(mysqli_query($this->con_dist, $del)) {
                    if($res['duplicated'] < 2) {
                        $this->delProjMongo($pid);
                        $this->chkDupProjs($pid, $res['domain']);
                    }

                    $result = array(
                        'response' => "Project successfully deleted.",
                        'response_code' => 100
                    );
                    return json_encode($result);
                } else {
                    $this->common->logErrorFatal('', 'mysql:', mysqli_error($this->con_dist), print_r(error_get_last(), true));
                }
            }
        } else {
            $this->common->logErrorFatal('', 'mysql:', mysqli_error($this->con_dist), print_r(error_get_last(), true));
        }
    }


    public function delProjMongo($pid)
    {
        $mongo = $this->dbc->conMongo();
        if (!is_object($mongo)) {
            $this->common->logErrorFatal('', 'mongo:', 'mongo connection object is unusable'.print_r($mongo, true), print_r(error_get_last(), true));
            return;
        }

        if(!empty($pid)) {
            $pid = (int)$pid;

            $mongo->auditor->remove(array('p_id' => $pid));
            $mongo->auditor_nonindex->remove(array('p_id' => $pid));
            $mongo->auditor_non200->remove(array('p_id' => $pid));
            $mongo->auditor_failed->remove(array('p_id' => $pid));
        }
    }


    public function chkDupProjs($pid, $domain)
    {
        $qry = "SELECT p_id,pages_to_crawl,duplicated,crawl_no FROM sa_projects WHERE domain='".$domain."' AND p_id != '".$pid."' AND expired=0  ORDER BY pages_to_crawl DESC LIMIT 1";
        if($chk = mysqli_query($this->con_dist, $qry)) {
            if(mysqli_num_rows($chk)) {
                $res = mysqli_fetch_assoc($chk);

                if($res['duplicated'] < 2) {
                    $this->markNominee($res['p_id'], $domain);
                }
            }
        } else {
            $this->common->logErrorFatal('', 'mysql:', mysqli_error($this->con_dist), print_r(error_get_last(), true));
        }
    }


    public function markNominee($pid, $domain)
    {
        $qry = "UPDATE sa_projects SET runnow=0,cstatus=0,duplicated=0,history_created=0,duplicated=0 WHERE domain='".$domain."' AND p_id = $pid";
        if(mysqli_query($this->con_dist, $qry)) {
            $qry2 = "UPDATE sa_projects SET cstatus=8,duplicated=2,history_created=2,dump_deleted=2,duplicated=0 WHERE p_id != $pid AND domain='".$domain."'";
            if(!mysqli_query($this->con_dist, $qry2)) {
                $this->common->logErrorFatal('', 'mysql:', mysqli_error($this->con_dist), print_r(error_get_last(), true));
            }
        } else {
            $this->common->logErrorFatal('', 'mysql:', mysqli_error($this->con_dist), print_r(error_get_last(), true));
        }
    }
// update mongo pid and weekno


    /**
     * Service to create runnow in Website Auditor
     *
     * Accepts GET params only
     *
     * @param int user_id
     * @param int pages_to_crawl (new quota for project)
     * @param string p_id
     * @param string project (domain_name)
     * @param int user_class (1=>new free, 2=>paid, 3=>free)
     * @param int project_class (1=>new, 2=>quota_increase, 3=>weekly)
     *
     * Return Value:
     *   JSON encoded array,
     *   VALUES:
     *     response: response message
     *     response_code: 100(success)/101(error)
     */
    public function QuotaIncrease($data)
    {
        $resp_code = 101;

        if(!isset($data['user_id']) || !isset($data['pages_to_crawl']) || !isset($data['p_id']) || !isset($data['project'])|| !isset($data['user_class'])|| !isset($data['project_class'])) {

            $this->common->logErrorFatal('', 'service:', print_r($data, true), print_r(error_get_last(), true));

            $result = array(
                'response' => 'Incomplete Data',
                'response_code' => $resp_code
            );

            return json_encode($result);
        }

        $user_id = $data['user_id'];
        $pages_to_crawl = (int)$data['pages_to_crawl'];
        $p_id = $data['p_id'];
        $p_class = $data['project_class'];
        $u_class = $data['user_class'];

        // $new_quota = $this->chkQuotaLimit($user_id, $p_id, $pages_to_crawl, $p_class);

        $priority =  $this->calculatePriority($u_class, $p_class);

        $qry = "UPDATE sa_projects SET runnow=0,pages_to_crawl=$pages_to_crawl,pages_crawled=$pages_to_crawl,expired=0 WHERE p_hash='".$p_id."' AND user_id=$user_id";

        if(mysqli_query($this->con_dist, $qry)) {
            if(mysqli_affected_rows($this->con_dist)) {
                $msg = "Quota changed successfully";
                $resp_code = 100;
            } else {
                $this->common->logErrorFatal('', 'service:no row affected in quota change', $qry.":".mysqli_error($this->con_dist), print_r(error_get_last(), true));

                $msg = "Unable to change Quota";
            }
        } else {
            $this->common->logErrorFatal('', "service:$qry", mysqli_error($this->con_dist), print_r(error_get_last(), true));

            $msg = "Unable to change Quota";
        }

        $result = array(
            'response' => $msg,
            'response_code' => $resp_code
        );
        return json_encode($result);
    }


    /**
     * Service to create runnow in Website Auditor
     *
     * Accepts GET params only
     *
     * @param int user_id
     * @param int pages_to_crawl (new quota for project)
     * @param string p_id
     * @param string project (domain_name)
     * @param int user_class (1=>new free, 2=>paid, 3=>free)
     * @param int project_class (1=>new, 2=>quota_increase, 3=>weekly)
     *
     * Return Value:
     *   JSON encoded array,
     *   VALUES:
     *     response: response message
     *     response_code: 100(success)/101(error)
     */
   public function QuotaDecrease($data)
    {
        $resp_code = 101;

        if(!isset($data['user_id']) || !isset($data['pages_to_crawl']) || !isset($data['p_id']) || !isset($data['project'])|| !isset($data['user_class'])|| !isset($data['project_class'])) {

            $this->common->logErrorFatal('', 'service:', print_r($data, true), print_r(error_get_last(), true));

            $result = array(
                'response' => 'Incomplete Data',
                'response_code' => $resp_code
            );

            return json_encode($result);
        }

        $user_id = $data['user_id'];
        $pages_to_crawl = (int)$data['pages_to_crawl'];
        $p_id = $data['p_id'];
        $p_class = $data['project_class'];
        $u_class = $data['user_class'];

        $priority =  $this->calculatePriority($u_class, $p_class);

        $qry = "UPDATE sa_projects SET pages_to_crawl=$pages_to_crawl,pages_crawled=$pages_to_crawl,expired=0 WHERE p_hash='".$p_id."' AND user_id=$user_id";

        if(mysqli_query($this->con_dist, $qry)) {
            if(mysqli_affected_rows($this->con_dist)) {
                $msg = "Quota changed successfully";
                $resp_code = 100;
            } else {
                $this->common->logErrorFatal('', 'service:no row affected in quota change', $qry.":".mysqli_error($this->con_dist), print_r(error_get_last(), true));

                $msg = "Unable to change Quota";
            }
        } else {
            $this->common->logErrorFatal('', "service:$qry", mysqli_error($this->con_dist), print_r(error_get_last(), true));

            $msg = "Unable to change Quota";
        }

        $result = array(
            'response' => $msg,
            'response_code' => $resp_code
        );
        return json_encode($result);
    }


    /**
     * Service to change project setting in Website Auditor
     *
     * Accepts GET params only
     *
     * @param string p_id
     * @param int setting (user_agent or follow_robot)
     * @param int setting_value (0=>off, 1=>on In case of follow_robot), (string in Case of user_agent)
     *
     * Return Value:
     *   JSON encoded array,
     *   VALUES:
     *     response: response message
     *     response_code: 100(success)/101(error)
     */
    public function setProjSetting($data)
    {
        $set_arr = array();

        $pid = $this->getIdFromHash($data['p_id']);

        if(!isset($data['p_id']) || !isset($data['robot_value']) || !isset($data['agent_value']) || !isset($pid)) {

            $this->common->logErrorFatal('', 'service:', print_r($data, true), print_r(error_get_last(), true));

            $result = array(
                'response' => 'Incomplete Data',
                'response_code' => 101
            );

            return json_encode($result);
        }

        $set_arr['follow_robot'] = $data['robot_value'];
        $set_arr['user_agent'] = rawurldecode($data['agent_value']);

        $new_setting = serialize($set_arr);

        if($s_data = $this->chkProjSetting($pid)) {
            $qry = "UPDATE proj_settings SET setting = '".$new_setting."', changed = 1, updated_on = NOW() WHERE p_id=$pid";
        } else {
            $qry = "INSERT INTO proj_settings(p_id,setting,changed,created_on,updated_on) VALUES($pid, '".$new_setting."', 1, NOW(), NOW())";
        }

        if(mysqli_query($this->con_dist, $qry)) {
            $result = array(
                'response' => "setting changed successfully",
                'response_code' => 100
            );
        } else {
            $this->common->logErrorFatal('', 'service:', mysqli_error($this->con_dist), print_r(error_get_last(), true));

            $result = array(
                'response' => "Internal Error",
                'response_code' => 101
            );
        }
        return json_encode($result);

    }


    /**
     * Service to change project setting in Website Auditor
     *
     * Accepts GET params only
     *
     * @param string p_id
     *
     * Return Value:
     *   JSON encoded array,
     *   VALUES:
     *     response: response message
     *     response_code: 100(success)/101(error)
     */
    public function getProjSetting($data)
    {
        $resp_code = 101;
        $data_arr = array();

        $pid = $this->getIdFromHash($data['p_id']);

        if(!isset($data['p_id']) || !isset($data['user_id']) || !isset($pid)) {

            $this->common->logErrorFatal('', 'service:', print_r($data, true), print_r(error_get_last(), true));

            $result = array(
                'response' => 'Incomplete Data',
                'response_code' => $resp_code
            );

            return json_encode($result);
        }

        $tot_used = $this->getAllProjPages($data['user_id'], $pid);
        $tot_proj = $this->getUserProjCount($data['user_id'], $pid);

        $u_data_arr['name'] = "total_quota";
        $u_data_arr['setting'] = $tot_used;
        $data_arr[] = $u_data_arr;

        $p_data_arr['name'] = "total_projects";
        $p_data_arr['setting'] = $tot_proj;
        $data_arr[] = $p_data_arr;

        $qry = "SELECT pages_to_crawl FROM sa_projects WHERE p_id = $pid";
        if($chk = mysqli_query($this->con_dist, $qry)) {
            while($res = mysqli_fetch_assoc($chk)) {
                $p_data_arr['name'] = "proj_quota";
                $p_data_arr['setting'] = $res['pages_to_crawl'];
                $data_arr[] = $p_data_arr;
            }
        } else {
            $this->common->logErrorFatal('', "service:$qry", mysqli_error($this->con_dist), print_r(error_get_last(), true));
        }


        if($s_data = $this->chkProjSetting($pid)) {
            $s_arr = unserialize($s_data['setting']);
            foreach ($s_arr as $key => $value) {
                $s_f_arr['name'] = $key;
                $s_f_arr['setting'] = $value;
                $data_arr[] = $s_f_arr;
            }
        } else {
            $qry = "SELECT name,setting FROM settings WHERE name = 'user_agent' OR name = 'follow_robot'";
            if($chk = mysqli_query($this->con_dist, $qry)) {
                while($res = mysqli_fetch_assoc($chk)) {
                    $data_arr[] = $res;
                }
            } else {
                $this->common->logErrorFatal('', "service:$qry", mysqli_error($this->con_dist), print_r(error_get_last(), true));
            }
        }

        $result = array(
            'response' => "setting data",
            'settings' => $data_arr,
            'response_code' => 100
        );
        return json_encode($result);

    }


    public function getIdFromHash($id_hash)
    {
        $qry = "SELECT p_id FROM sa_projects WHERE p_hash='".$id_hash."'";

        if($chk = mysqli_query($this->con_dist, $qry)) {
            var_dump($chk);
            if(mysqli_num_rows($chk)) {
                $row = mysqli_fetch_assoc($chk);
                return $row['p_id'];
            }
        } else {
            $this->common->logErrorFatal('', "service:$qry", mysqli_error($this->con_dist), print_r(error_get_last(), true));
        }
        return false;
    }


    public function chkProjSetting($pid)
    {
        $qry = "SELECT id,setting FROM proj_settings WHERE p_id=$pid";

        if($chk = mysqli_query($this->con_dist, $qry)) {
            if(mysqli_num_rows($chk)) {
                $row = mysqli_fetch_assoc($chk);
                return $row;
            }
        } else {
            $this->common->logErrorFatal('', "service:$qry", mysqli_error($this->con_dist), print_r(error_get_last(), true));
        }
        return false;
    }


    /**
     * Service to project servers in Website Auditor
     *
     * Accepts GET params only
     *
     * @param string p_id, first_stat, action=get_proj_servers
     *
     * Return Value:
     *   JSON encoded array,
     *   VALUES:
     *     response: response message
     *     response_code: 100(success)/101(error)
     */
    public function getProjServers($data)
    {
        $master = "";
        $slaves = array();
        $res = array();
        $pages = array();
        $new_runnow = 0;
        $res['new_runnow'] = 0;
        $time_diff = 10;
        $status = 0;
        $time_diff_sec = 0;

        $qry = "SELECT p_id,cstatus,master_id,runnow,ext_runnow,pages_to_crawl,cauditor_id,pauditor_id,crawl_no,duplicated FROM sa_projects WHERE p_hash='".$data['p_id']."'";

        if($chk = mysqli_query($this->con_dist, $qry)) {
            if(mysqli_num_rows($chk)) {
                $row = mysqli_fetch_assoc($chk);
                $row_res = $row;
                $pid = $row['p_id'];
                $master = $row['master_id'];
                $runnow = $row['runnow'];
                if ($row['ext_runnow'] < 2)
                    $auditor_id = $row['pauditor_id'];
                else
                    $auditor_id = $row['cauditor_id'];

                $ext_runnow = $row['ext_runnow'];
                $pages_to_crawl = $row['pages_to_crawl'];
                if(!empty($row['cauditor_id'])) {
                    $time_diff = (abs($row['cauditor_id'] - time()) / 86400);
                    $time_diff_hour = $time_diff * 24;
                    $time_diff_min = $time_diff_hour * 60;
                    $time_diff_sec = $time_diff_min * 60;
                }
            }
        } else {
            $this->common->logErrorFatal('', "service:$qry", mysqli_error($this->con_dist), print_r(error_get_last(), true));
        }


        $qry = "SELECT id FROM servers WHERE type=1";
        if($chk = mysqli_query($this->con_dist, $qry)) {
            if(mysqli_num_rows($chk)) {
                while($row2 = mysqli_fetch_assoc($chk)) {
                    $slaves[] = $row2['id'];
                }
            }
        } else {
            $this->common->logErrorFatal('', "service:$qry", mysqli_error($this->con_dist), print_r(error_get_last(), true));
        }

        if(empty($master)) {
            $master = "";
        }

        if(!isset($data['runnow_request'])) {
            $res['master'] = $master;
            $res['slaves'] = $slaves;
            $res['runnow'] = $runnow;
            $res['pages_to_crawl'] = $pages_to_crawl;
        }
        $time_remaining = abs((5*60) - $time_diff_sec);
        if(isset($quota_runnow) && $quota_runnow == 1 && ($time_remaining * 60) <= 5) {
            $res['quota_duration'] = $time_remaining;
        }

        if ($row['runnow'] == '2' && (!$row['cstatus'] || $row['cstatus'] > '2' ))
        {
            if ( !$row['ext_runnow'] || ($row['ext_runnow'] == '2' && (!$row['cstatus'] || $row['cstatus'] > '2' )) )
                $res['completed'] = 1;
            else
                $res['completed'] = 0;
        }
        else
            $res['completed'] = 0;

        if ($row['runnow'] == '2' && $row['duplicated'] == '2' && $row['ext_runnow'])
            $res['old_completed'] = 1;
        if ( $row['runnow'] < '2' )
            $res['new_runnow'] = 1;

        $res['pages'] = array();


        $result = array(
                    'response' => "server details",
                    'servers' => $res,
                    'response_code' => 100
                );
        return json_encode($result);

        
        if($new_runnow == 1) {
            $qry = "UPDATE sa_projects SET runnow=1,cstatus=0 WHERE p_id=".$pid;
            if(!mysqli_query($this->con_dist, $qry)) {
                $this->common->logErrorFatal('', "service:$qry", mysqli_error($this->con_dist), print_r(error_get_last(), true));
            } else {
                

            }
        }

        $mongo = $this->dbc->conMongo();
        if (!is_object($mongo)) {
            $this->common->logErrorFatal('', 'mongo:', 'mongo connection object is unusable'.print_r($mongo, true), print_r(error_get_last(), true));
        }
        $m_pages = $mongo->auditor->find(array('p_id' => (int)$pid), array('_id' => 0, 'url' => 1));
        $m_pages->limit(100);
        while($pgs = $m_pages->getNext()) {
            $pages[] = $pgs['url'];
        }
        if(count($pages) < 100) {
            $m_pages = $mongo->auditor_non200->find(array('p_id' => (int)$pid), array('_id' => 0, 'url' => 1));
            $m_pages->limit(100);
            while($pgs = $m_pages->getNext()) {
                $pages[] = $pgs['url'];
            }
        }
        if(count($pages) < 100) {
            $m_pages = $mongo->auditor_nonindex->find(array('p_id' => (int)$pid), array('_id' => 0, 'url' => 1));
            $m_pages->limit(100);
            while($pgs = $m_pages->getNext()) {
                $pages[] = $pgs['url'];
            }
        }
        if(count($pages) < 100) {
            $m_pages = $mongo->auditor_failed->find(array('p_id' => (int)$pid), array('_id' => 0, 'url' => 1));
            $m_pages->limit(100);
            while($pgs = $m_pages->getNext()) {
                $pages[] = $pgs['url'];
            }
        }
        $res['pages'] = $pages;
        

        if($res['completed'] == 0) {
            $q_status = $this->checkQueueTimeStatus($data['p_id'], $row_res['runnow'], $row_res['cstatus']);
            if(!empty($q_status[0])) {

                $res['no_in_queue'] = $q_status[0];
                $res['time_in_queue'] = ceil((int)$q_status[1] * 0.48);
            }

        }

        $result = array(
            'response' => "server details",
            'servers' => $res,
            'response_code' => 100
        );
        return json_encode($result);
    }


    public function checkQueueTimeStatus($id_hash, $runnow, $c_status)
    {
        $qry = "";
        $qry_add = "";
        $qry_add2 = "";
        $no_in_queue = 0;
        $pages_in_queue = 0;

        if($c_status == 0) {
            $qry = "SELECT p_hash,pages_to_crawl FROM sa_projects WHERE ((last_crawled_on < NOW() - INTERVAL 1 WEEK AND crawled='2') OR (last_crawled_on IS NULL AND crawled='0') OR (status='2' AND runnow_priority=98)) AND runnow='1' AND expired=0 ORDER BY runnow_priority ASC";

            if($runnow == 0) {
                $qry_add = "SELECT p_hash,pages_to_crawl FROM sa_projects WHERE ((last_crawled_on < NOW() - INTERVAL 1 WEEK AND crawled='2') OR (last_crawled_on IS NULL AND crawled='0')) AND expired=0 AND user_id IN (".PRIORITY_USERS.")";
            }
        }
        elseif($c_status == 0 || $c_status == 5) {
            $qry_add = "SELECT DISTINCT a.p_hash,a.pages_to_crawl FROM sa_projects a LEFT JOIN work_progress b ON a.id = b.p_id WHERE a.cstatus IN ('1','2','0','5') AND a.runnow=1 AND b.scraped IN (1,2) ORDER BY a.id";

            if($runnow == 0) {
                $qry = "SELECT DISTINCT a.p_hash,a.pages_to_crawl FROM sa_projects a LEFT JOIN work_progress b ON a.id = b.p_id WHERE a.cstatus IN ('1','2','0','5') AND a.dup_running=97 AND b.scraped IN (1,2) ORDER BY a.id";
                $qry_add2 = "SELECT DISTINCT a.p_hash,a.pages_to_crawl FROM sa_projects a LEFT JOIN work_progress b ON a.id = b.p_id WHERE a.cstatus IN ('1','2','0','5') AND a.runnow=0 AND b.scraped IN (1,2) ORDER BY a.id";
            }
        }

        $qry_arr = array($qry, $qry_add, $qry_add2);

        foreach ($qry_arr as $qry_str) {
            $chk = "";
            $row = array();

            if(!empty($qry_str)) {
                if($chk = mysqli_query($this->con_dist, $qry_str)) {
                    if(mysqli_num_rows($chk)) {
                        while($row = mysqli_fetch_assoc($chk)) {
                            if($id_hash !== $row['p_hash']) {
                                $no_in_queue++;
                                $pages_in_queue += $row['pages_to_crawl'];
                            }
                            elseif($id_hash === $row['p_hash']) {
                                return array($no_in_queue, $pages_in_queue);
                            }
                        }
                    }
                } else {
                    $this->common->logErrorFatal('', "service:$qry", mysqli_error($this->con_dist), print_r(error_get_last(), true));
                }
            }
        }
        return array($no_in_queue, $pages_in_queue);
    }

    public function getSlaveIp($s_ids)
    {
        $s_arr = array();
        $qry = "SELECT id,ip FROM servers WHERE id IN ($s_ids)";
        if($chk = mysqli_query($this->con_dist, $qry)) {
            if(mysqli_num_rows($chk)) {
                while($row = mysqli_fetch_assoc($chk)) {
                    $slaves = array();
                    $slaves['id'] = $row['id'];
                    $slaves['ip'] = $row['ip'];
                    $s_arr[] = $slaves;
                }
            }
        } else {
            $this->common->logErrorFatal('', "service:$qry", mysqli_error($this->con_dist), print_r(error_get_last(), true));
        }
        return $s_arr;
    }


    public function getPythonData($runnow, $master_ip, $pid)
    {
        $args = "$pid $master_ip $runnow";
        $file = "progress_checker.py $args";
        $last_l = exec("python progress_checker.py $args", $out, $err);
        return json_decode($out[0]);
    }


    /**
     * Service to project progress in Website Auditor
     *
     * Accepts GET params only
     *
     * @param string p_id
     * @param string slave
     * @param string runnow
     * @param string master
     * @param int last_count
     * @param string action=get_proj_progress
     *
     * Return Value:
     *   JSON encoded array,
     *   VALUES:
     *     response: response message
     *     response_code: 100(success)/101(error)
     */
    public function getProjProgress($data)
    {   
        $res = array();
        $masters = array();
        $unprocessed = 0;
        $res['completed'] = 0;

        $runnow = $data['runnow'];
        $c_total = $data['last_crawled_pages'];
        $c_processed = $data['last_analyzed_pages'];
    
        $p_data = $this->getProjData($data['p_id']);
        if ($data['master'] == '')
        {
            $queued_projects = $this->get_queued_projects();
            $queue_no = $this->get_queue_no($queued_projects,$data['p_id']);
            $res['crawl_completed'] = 0;
            $res['no_in_queue'] = $queue_no + 1;
            $res['pages'] = array();
            $res['completed'] = 0;
            $res['crawled'] = 0;
            $res['pages_to_crawl'] = $p_data['pages_to_crawl'];

            $result = array(
            'response' => "currently processing",
            'progress' => $res,
            'response_code' => 100
            );
            return json_encode($result);
        }
        $pid = (int)$p_data['p_id'];
        if($p_data['dup_running'] == 1) {
            $dqry = "SELECT p_id FROM sa_projects WHERE domain='".$p_data['domain']."' AND duplicated<2";
            $dchk = mysqli_query($this->con_dist, $dqry);
            $dres = mysqli_fetch_assoc($dchk);
            $pid = (int)$dres['p_id'];
        }

        $s_str = implode(",", $data['slaves']).",".$data['master'];
        $slave_ips = $this->getSlaveIp($s_str);

        $py_data = 0;
        foreach ($slave_ips as $value) {
            if(in_array($value['id'], explode(",", $data['master']))) {
                $py_data += $this->getPythonData($runnow, $value['ip'], $pid);
            }
        }
        $res['crawled'] = $py_data;

        $p_found = $this->getProgressUrl($slave_ips, $pid, $c_total, $c_processed);
        array_reverse($p_found);

        if(empty($p_found)) {
            $p_found = array();
        }
        $res['pages'] = $p_found;
        //crawled,scraped,processed
        //$qry = "SELECT cstatus,master_id,runnow,pages_to_crawl,status,runnow,crawl_no,runnow_priority,duplicated FROM sa_projects WHERE id='".$pid."'";

        $qry = "SELECT cstatus,master_id,runnow,pages_to_crawl,runnow,crawl_no,duplicated,ext_runnow FROM sa_projects WHERE p_id='".$pid."'";
        
        if($chk = mysqli_query($this->con_dist, $qry)) {
            if(mysqli_num_rows($chk)) {
                $row = mysqli_fetch_assoc($chk);

                $row_res = $row;
                $n_p_to_crawl = $row['pages_to_crawl'];
                if($py_data == 0 || $row['cstatus'] == 2) {
                    $res['crawl_completed'] = 1;
                }

                if ( $row['runnow'] == '2' && (!$row['cstatus'] || $row['cstatus'] > '2' ) )
                {
                    if ( !$row['ext_runnow'] || ($row['ext_runnow'] == '2' && (!$row['cstatus'] || $row['cstatus'] > '2' )) )
                        $status = 1;
                    else
                        $status = 0;
                }
                else
                    $status = 0;

                if ($row['runnow'] == '2' && $row['duplicated'] == '2' && $row['ext_runnow'])
                    $old_status = 1;
            }
        } else {
            $this->common->logErrorFatal('', "service:$qry", mysqli_error($this->con_dist), print_r(error_get_last(), true));
        }


        if((empty($p_found) || $unprocessed < 2) && $status == 1) {
            $res['completed'] = 1;
        } else {
            $res['completed'] = 0;
        }
        if(isset($old_status) && $old_status == 1) {
            $res['old_completed'] = 1;
        }
        $res['pages_to_crawl'] = $n_p_to_crawl;

        if($res['completed'] == 0) {
            $q_status = $this->checkQueueTimeStatus($data['p_id'], $row_res['runnow'], $row_res['cstatus']);
            if(!empty($q_status[0])) {

                $res['no_in_queue'] = $q_status[0];
                $res['time_in_queue'] = ceil((int)$q_status[1] * 0.48);
            }

        }

        $result = array(
            'response' => "currently processing",
            'progress' => $res,
            'response_code' => 100
        );
        return json_encode($result);
    }


    public function getProgressUrl($slave_ips, $pid, $c_total, $c_processed)
    {
        $pages = array();
        $processed = 0;
        foreach ($slave_ips as $value) {
            $con_temp = new mysqli($value['ip'], $this->temp_user, $this->temp_pass, $this->temp_db);
            if (mysqli_connect_error()) {
                $this->common->logErrorFatal('', "connection:", mysqli_connect_error(), print_r(error_get_last(), true));
            }

            $qry = "SELECT u,processed FROM temp WHERE pi=$pid ORDER BY id Desc";
            if($chk = mysqli_query($con_temp, $qry)) {
                if(mysqli_num_rows($chk)) {
                    while($row = mysqli_fetch_assoc($chk)){
                        if($row['processed'] == 1) {
                            $processed += 1;
                        }
                        $pages[] = $row;
                    }
                }
            } else {
                $this->common->logErrorFatal('', "service:$qry", mysqli_error($con_temp), print_r(error_get_last(), true));
            }

            // if(!empty($pages) && (count($pages) >= 100)) {
            //     break;
            // }
        }
        $mongo = $this->dbc->conMongo();
        if (!is_object($mongo)) {
            $this->common->logErrorFatal('', 'mongo:', 'mongo connection object is unusable'.print_r($mongo, true), print_r(error_get_last(), true));
        }
        // if(count($pages) < $c_total) {
            $m_pages = $mongo->auditor->find(array('p_id' => (int)$pid), array('_id' => 0, 'url' => 1));
            // $m_pages->limit(100);
            while($pgs = $m_pages->getNext()) {
                $row = array();
                $row['url'] = $pgs['url'];
                $row['processed'] = 1;
                $pages[] = $row;
                $processed += 1;

                // if(!empty($pages) && (count($pages) >= 100)) {
                //     break;
                // }
            }
        // }
        // if(count($pages) < $c_total) {
            $m_pages = $mongo->auditor_non200->find(array('p_id' => (int)$pid), array('_id' => 0, 'url' => 1));
            // $m_pages->limit(100);
            while($pgs = $m_pages->getNext()) {
                $row = array();
                $row['url'] = $pgs['url'];
                $row['processed'] = 1;
                $pages[] = $row;
                $processed += 1;
                // if(!empty($pages) && (count($pages) >= 100)) {
                //     break;
                // }
            }
        // }
        // if(count($pages) < $c_total) {
            $m_pages = $mongo->auditor_nonindex->find(array('p_id' => (int)$pid), array('_id' => 0, 'url' => 1));
            // $m_pages->limit(100);
            while($pgs = $m_pages->getNext()) {
                $row = array();
                $row['url'] = $pgs['url'];
                $row['processed'] = 1;
                $pages[] = $row;
                $processed += 1;
                // if(!empty($pages) && (count($pages) >= 100)) {
                //     break;
                // }
            }
        // }
        // if(count($pages) < $c_total) {
            $m_pages = $mongo->auditor_failed->find(array('p_id' => (int)$pid), array('_id' => 0, 'url' => 1));
            // $m_pages->limit(100);
            while($pgs = $m_pages->getNext()) {
                $row = array();
                $row['url'] = $pgs['url'];
                $row['processed'] = 1;
                $pages[] = $row;
                $processed += 1;
                // if(!empty($pages) && (count($pages) >= 100)) {
                //     break;
                // }
            }
        // }
        return $pages;
    }


    public function getProjData($id_hash)
    {
        $res = array();
        $qry = "SELECT * FROM sa_projects WHERE p_hash='".$id_hash."'";

        if($chk = mysqli_query($this->con_dist, $qry)) {
            $res = mysqli_fetch_assoc($chk);
        } else {
            $this->common->logErrorFatal('', "service:$qry", mysqli_error($con_dist), print_r(error_get_last(), true));
        }
        return $res;
    }


    public function chkDupRunning($id_hash)
    {
        $qry = "SELECT p_id,duplicated,domain FROM sa_projects WHERE p_hash='".$id_hash."'";
        if($chk = mysqli_query($this->con_dist, $qry)) {
            if(mysqli_num_rows($chk)) {
                $row = mysqli_fetch_assoc($chk);

                if($row['duplicated'] < 2) {
                    $qry2 = "SELECT p_id FROM sa_projects WHERE domain='".$row['domain']."' AND duplicated<2";
                    if($chk2 = mysqli_query($this->con_dist, $qry2)) {
                        if(mysqli_num_rows($chk2)) {
                            $row2 = mysqli_fetch_assoc($chk2);
                            $send_pid = $row2['p_id'];
                        } else {
                            $this->convertToRunnow($row['p_id']);
                            $send_pid = $row['p_id'];
                        }
                    } else {
                        $this->common->logErrorFatal('', "service:$qry", mysqli_error($con_dist), print_r(error_get_last(), true));
                    }
                } else {
                    return $row['p_id'];
                }
            }
        } else {
            $this->common->logErrorFatal('', "service:$qry", mysqli_error($con_dist), print_r(error_get_last(), true));
        }
    }


    public function convertToRunnow($pid)
    {
        $qry = "UPDATE sa_projects SET runnow=0,cstatus=0 WHERE p_id=".$pid;
        if(!mysqli_query($this->con_dist, $qry)) {
            $this->common->logErrorFatal('', "service:$qry", mysqli_error($con_dist), print_r(error_get_last(), true));
        }
    }


    public function weeklyEmail($data)
    {
        $res_arr = array();

        $proj_data_arr = $this->chkProjData($data['user_id']);
        $chk_proj_data = $proj_data_arr[0];

        $mongo = $this->dbc->conMongo();
        if (!is_object($mongo)) {
            $this->common->logErrorFatal('', 'mongo:', 'mongo connection object is unusable'.print_r($mongo, true), print_r(error_get_last(), true));
            return;
        }

        foreach ($proj_data_arr as $pjkey => $pjvalue) {
            $pid = (int)$pjvalue['p_id'];
            $week = ((int)$pjvalue['crawl_no'] - 1);

            $totalScraped = $mongo->auditor->count(
                array('p_id' => $pid, 'w' => $week)
            );

            $totalDups = $mongo->auditor->count(
                array('p_id' => $pid, 'w' => $week, 'tu' => array('$gt' => 0))
            );

            $totalUnique = $totalScraped - $totalDups;

            $tCountNon200 = $mongo->auditor_non200->count(
                array('p_id' => $pid, 'w' => $week)
            );

            $tCountNon200_ni = $mongo->auditor_nonindex->count(
                array('p_id' => $pid, 'w' => $week)
            );

            $tCountFailed = $mongo->auditor_failed->count(
                array('p_id' => $pid, 'w' => $week)
            );

            if($pjvalue['runnow_priority'] == 99) {
                $red_data = $this->getReducesData($totalUnique, $totalDups, $tCountNon200, $tCountNon200_ni, $tCountFailed, $pjvalue['pages_to_crawl']);

                if(!empty($red_data['unique'])) {
                    $totalUnique = $red_data['unique'];
                }
                if(!empty($red_data['duplicate'])) {
                    $totalDups = $red_data['duplicate'];
                }
                if(!empty($red_data['non200'])) {
                    $tCountNon200 = $red_data['non200'];
                }
                if(!empty($red_data['nonindex'])) {
                    $tCountNon200_ni = $red_data['nonindex'];
                }
                if(!empty($red_data['failed'])) {
                    $tCountFailed = $red_data['failed'];
                }
            }
            if($pjkey == 0) {
                $section = "main";
            } else {
                $section = "additional";
            }

            if($section == "main") {
                $err_major = 0;
                $err_medium = 0;
                $err_minor = 0;
                $e_err_arr = array();

                $desc_arr = array(
                    'mal' => array('Missing Image Alt', 'have missing ALT in images', 'eimg.mal'),
                    'mkw' => array('Missing Meta Keyword', 'have meta keywords missing', 'emet.mkw'),
                    'lkw' => array('Long Meta Keyword', 'have too long meta keyword', 'emet.lkw'),
                    'skw' => array('Short Meta Keyword', 'have too short meta keyword', 'emet.skw'),
                    'mds' => array('Missing Meta Description', 'have meta description missing', 'emet.mds'),
                    'lds' => array('Long Meta Description', 'have too long meta description', 'emet.lds'),
                    'sds' => array('Short Meta Description', 'have too short meta keyword', 'emet.sds'),
                    'thn' => array('Thin Content', 'have thin content', 'epage.thn'),
                    'low' => array('Text to Html Ratio', 'have low text to html ratio', 'etth.low'),
                    'dti' => array('Duplicate Title', 'have duplicate title in them', 'etit.dti'),
                    'dkw' => array('Duplicate Meta Keywords', 'have duplicate meta keywords in them', 'emet.dkw'),
                    'dde' => array('Duplicate Meta Description', 'have duplicate meta description in them', 'emet.dde'),
                    'tu' => array('Duplicate Content', 'have duplicate content in them', 'tu'),
                    'lt' => array('Long Title', 'have too long title', 'etit.lt'),
                    'sti' => array('Short Title', 'have too short title', 'etit.sti'),
                    'eti' => array('Empty/Missing Title', 'have empty title', 'etit.eti'),
                    'mrf' => array('Meta Refresh', 'have meta refresh', 'emet.mrf'),
                    'mre' => array('Meta Redirect', 'have meta redirect', 'emet.mre'),
                    'mnf' => array('Meta Nofollow', 'have meta nofollow', 'emet.mnf'),
                    'mlg' => array('Missing Language Declaration', 'have language declaration missing', 'mlg'),
                    'mst' => array('Too Many Stylesheets', 'have too many stylesheets included', 'esht.mst'),
                    'mh1' => array('Missing H1 Tag', 'have missing h1 tag', 'eh1s.mh1'),
                    'eh1' => array('Too Many H1 Tags', 'have too many h1 tags', 'eh1s.eh1'),
                    'mh2' => array('Missing H2 Tag', 'have missing h2 tag', 'eh2s.mh2'),
                    'eh2' => array('Too Many H2 Tags', 'have too many h2 tags', 'eh2s.eh2'),
                    'lu' => array('Long URL', 'have too long URL', 'eurl.lu'),
                    'spu' => array('Special Characters in URL', 'have special characters in URL', 'eurl.spu'),
                    'unu' => array('Underscore Characters in URL', 'have underscores in URL', 'eurl.unu'),
                    'upu' => array('UpperCase Characters in URL', 'have uppercase characters in URL', 'eurl.upu'),
                    'frm' => array('Using Iframes', 'have iframes in content', 'efrm.frm'),
                    'fls' => array('Using Flash', 'have flash in content', 'efls.fls'),
                    'ent' => array('Too Many Internal Links', 'have too many internal links', 'eln.ent'),
                    'eex' => array('Too Many External Links', 'have too many external links', 'eln.eex'),
                    'nfc' => array('NoFollow Links', 'have nofollow links', 'eln.nfc'),
                    'eim' => array('Too many images', 'have too many images', 'eimg.eim'),
                    't' => array('Slow Page Speed', 'have slow response time', 't'),
                    'big' => array('Page Size too Big', 'have big page size', 'epage.big'),
                    'cms' => array('Missing Characterset Declaration', 'have characterset declaration missing', 'emet.cms'),
                    'dms' => array('Missing Doctype Declaration', 'have doctype declaration missing', 'edoc.dms')
                );

                foreach ($desc_arr as $eckey => $ecvalue) {
                    $err_col_arr[$ecvalue[2]] = 1;
                }


                $processed_arr = array(
                );

                $final_arr = array_merge($desc_arr, $processed_arr);

                $err_col_arr['_id'] = 0; $err_col_arr['pos'] = 1;

                $each_errors = $mongo->auditor->find(
                    array('p_id' => $pid, 'w' => $week),
                    $err_col_arr
                );
                $each_errors->limit((int)$pjvalue['pages_to_crawl']);

                $page_errors = array();

                while($e_error = $each_errors->getNext())
                {
                    $e_err_arr[] = $e_error;
                }
                foreach ($e_err_arr as $erval) {
                    foreach ($erval as $eskey => $esvalue) {
                        if($eskey !== "url" && $eskey !== "pos" && $eskey !== 't') {
                            if(is_array($esvalue)) {
                                foreach ($esvalue as $esk => $esv) {
                                    if(!empty($esvalue[$esk]) && $esvalue[$esk] >= 1 && $esk !== 'tit') {
                                        if(empty($page_errors[$esk])) {
                                            $page_errors[$esk] = 0;
                                        }
                                        $page_errors[$esk] += 1;
                                    }
                                }
                            }
                            elseif(!empty($erval[$eskey]) && $erval[$eskey] >= 1 && $eskey !== 'tit') {
                                if(empty($page_errors[$eskey])) {
                                    $page_errors[$eskey] = 0;
                                }
                                $page_errors[$eskey] += 1;
                            }
                        }

                    }

                    if(!empty($erval['t']) && $erval['t'] > T_SPEED_SLOW) {
                        if(empty($page_errors['t'])) {
                            $page_errors['t'] = 0;
                        }
                        $page_errors['t'] += 1;
                    }
                }
                foreach ($page_errors as $pkey => $pvalue) {
                    if(strstr(T_ERR_MAJOR, $pkey)) {
                        $err_major += 1;
                    }
                    elseif(strstr(T_ERR_MEDIUM, $pkey)) {
                        $err_medium += 1;
                    }
                    elseif(strstr(T_ERR_MINOR, $pkey)) {
                        $err_minor += 1;
                    }
                }
                $res_arr[$section][$pjvalue['p_hash']]['current'] = array('t_unique' => $totalUnique, 't_duplicate' => $totalDups, 't_non200' => $tCountNon200, 't_nonindex' => $tCountNon200_ni, 't_failed' => $tCountFailed, 't_major' => $err_major, 't_medium' => $err_medium, 't_minor' => $err_minor, 'domain' => $pjvalue['domain']);
            } else {
                $res_arr[$section][$pjvalue['p_hash']]['current'] = array('t_unique' => $totalUnique, 't_duplicate' => $totalDups, 't_non200' => $tCountNon200, 't_nonindex' => $tCountNon200_ni, 't_failed' => $tCountFailed);
            }

            $res_arr[$section][$pjvalue['p_hash']]['history'] = $this->getHistoryData($pid, $week, $pjvalue, $mongo);
        }


        $result = array(
            'response' => "Weekly Email Data",
            'progress' => $res_arr,
            'response_code' => 100
        );
        return json_encode($result);
    }


    public function chkProjData($user_id)
    {
        $proj_arr = array();

        // $qry = "SELECT id,domain,runnow_priority,pages_to_crawl,crawl_no,dup_running FROM projects WHERE id_hash='".$id_hash."'";
        // if($chk = mysqli_query($this->con_dist, $qry)) {
        //     $res = mysqli_fetch_assoc($chk);
        //     $final_arr['orig'] = $res;

        //     if($res['dup_running'] == 1) {
        //         $qry = "SELECT id,domain,runnow_priority,pages_to_crawl,crawl_no FROM projects WHERE domain='".$res['domain']."' AND dup_running=0";
        //         if($chk = mysqli_query($this->con_dist, $qry)) {
        //             $res = mysqli_fetch_assoc($chk);
        //             $final_arr['data'] = $res;
        //         } else {
        //             $this->common->logErrorFatal('', "service:$qry", mysqli_error($con_dist), print_r(error_get_last(), true));
        //         }
        //     } else {
        //         $final_arr['data'] = $res;
        //     }
        // } else {
        //     $this->common->logErrorFatal('', "service:$qry", mysqli_error($con_dist), print_r(error_get_last(), true));
        // }
        // return $final_arr;

        $qry = "SELECT p_id,p_hash,domain,pages_to_crawl,crawl_no FROM sa_projects WHERE user_id=$user_id AND duplicated<2 AND expired=0 ORDER BY pages_to_crawl DESC LIMIT 10";
        if($chk = mysqli_query($this->con_dist, $qry)) {
            while($res = mysqli_fetch_assoc($chk)) {
                $proj_arr[] = $res;
            }
        } else {
            $this->common->logErrorFatal('', "service:$qry", mysqli_error($con_dist), print_r(error_get_last(), true));
        }
        return $proj_arr;

    }


    function getHistoryData($pid, $week, $chk_proj_data, $mongo)
    {
        $week = $week - 1;
        $pid = (int)$pid;

        $totalScraped = $mongo->auditor->count(
            array('p_id' => $pid, 'w' => $week)
        );

        $totalDups = $mongo->auditor->count(
            array('p_id' => $pid, 'w' => $week, 'tu' => array('$gt' => 0))
        );

        $totalUnique = $totalScraped - $totalDups;

        $tCountNon200 = $mongo->auditor_non200->count(
            array('p_id' => $pid, 'w' => $week)
        );

        $tCountNon200_ni = $mongo->auditor_nonindex->count(
            array('p_id' => $pid, 'w' => $week)
        );

        $tCountFailed = $mongo->auditor_failed->count(
            array('p_id' => $pid, 'w' => $week)
        );

        if($chk_proj_data['runnow_priority'] == 99) {
            $red_data = $this->getReducesData($totalUnique, $totalDups, $tCountNon200, $tCountNon200_ni, $tCountFailed, $chk_proj_data['pages_to_crawl']);

            if(!empty($red_data['unique'])) {
                $totalUnique = $red_data['unique'];
            }
            if(!empty($red_data['duplicate'])) {
                $totalUnique = $red_data['duplicate'];
            }
            if(!empty($red_data['non200'])) {
                $tCountNon200 = $red_data['non200'];
            }
            if(!empty($red_data['nonindex'])) {
                $tCountNon200_ni = $red_data['nonindex'];
            }
            if(!empty($red_data['failed'])) {
                $tCountFailed = $red_data['failed'];
            }
        }
        $page_errors['t_unique'] = $totalUnique;
        $page_errors['t_duplicate'] = $totalDups;
        $page_errors['t_non200'] = $tCountNon200;
        $page_errors['t_nonindex'] = $tCountNon200_ni;
        $page_errors['t_failed'] = $tCountFailed;
        // $page_errors['t_crawled'] = $totalUnique + $totalDups + $tCountNon200 + $tCountNon200_ni + $tCountFailed;

        return $page_errors;
    }


    function getReducesData($totalUnique, $totalDups, $tCountNon200, $tCountNon200_ni, $tCountFailed, $total_pages)
    {
        $unique_count = 0;
        $dup_count = 0;
        $non200_count = 0;
        $nonindex_count = 0;
        $failed_count = 0;
        $pages_found = 0;
        while($pages_found <= $total_pages) {

            if($totalUnique > 0) {
                $pages_found++;
                $unique_count++;

                $totalUnique--;
                $total_pages--;
            }

            if($totalDups > 0) {
                $pages_found++;
                $dup_count++;

                $totalDups--;
                $total_pages--;
            }

            if($tCountNon200 > 0) {
                $pages_found++;
                $non200_count++;

                $tCountNon200--;
                $total_pages--;
            }

            if($tCountNon200_ni > 0) {
                $pages_found++;
                $nonindex_count++;

                $tCountNon200_ni--;
                $total_pages--;
            }

            if($tCountFailed > 0) {
                $pages_found++;
                $failed_count++;

                $tCountFailed--;
                $total_pages--;
            }
        }
        return array("unique"=>$unique_count,"duplicate"=>$dup_count,"non200"=>$non200_count,"nonindex"=>$nonindex_count,"failed"=>$failed_count);
    }

    function get_queued_projects()
    {
        $dqry = "SELECT p_hash FROM sa_projects WHERE runnow = 0 order By p_id ASC ";
        $dchk = mysqli_query($this->con_dist, $dqry);
        $dres = array();
        while ( $row = mysqli_fetch_assoc($dchk) )
        {
            $dres[] = $row;
        }
        return $dres;
    }

    function get_queue_no($queued_projects, $pid) 
    {
        foreach($queued_projects as $index => $project) 
        {
            if($project['p_hash'] == $pid) return $index;
        }
        return FALSE;
    }


}