<?php

require_once __DIR__.'/Common.php';

class Services
{
    private $common;
    private $con_dist;
    private $dbc;
    private $temp_db;
    private $temp_user;
    private $temp_pass;

    public function __construct()
    {
        $this->common = new Common();
        $this->dbc = new Database();

        $dist = $this->dbc->conDist();

        $this->temp_pass = "yb9738z";
        $this->temp_user = "auditor";
        $this->temp_db = "auditor_temp";

        if ($dist instanceof mysqli) {
            $this->con_dist = $dist;
        } else {
            $this->common->logErrorFatal('', 'mysql:', 'dist mysql connection object is unusable'.print_r($dist, true), print_r(error_get_last(), true));
        }
    }


    public function chkProjExists($pid)
    {
        $findProj = "SELECT id,domain,user_id,crawled,runnow FROM projects WHERE id_hash = '".$pid."'";

       if ($findP = mysqli_query($this->con_dist, $findProj)) {
            if(mysqli_num_rows($findP)) {
                $data = mysqli_fetch_assoc($findP);
                return $data;
            } else {
                return false;
            }
        } else {
            $this->common->logErrorFatal('', 'mysql:', mysqli_error($this->con_dist), print_r(error_get_last(), true));

            $result = array(
                'response' => 'Internal Error Occured',
                'response_code' => 101
            );

            echo json_encode($result);
            exit();
        }
    }


    public function calculatePriority($u_class, $p_class)
    {
        if($u_class == 1 && $p_class == 1) {
            $priority = 1;
        }
        elseif($u_class == 2 && $p_class == 1) {
            $priority = 2;
        }
        elseif($u_class == 2 && $p_class == 2) {
            $priority = 3;
        }
        elseif($u_class == 3 && $p_class == 2) {
            $priority = 4;
        }
        elseif($p_class == 3) {
            $priority = 5;
        }
        else {
            $priority = 5;
            // $this->common->logErrorFatal('', "service:incorrect priority params", "priority:u_class:$u_class,p_class:$p_class", print_r(error_get_last(), true));

            // $result = array(
            //     'response' => 'Incorect priority combination',
            //     'response_code' => 101
            // );

            // echo json_encode($result);
            // exit();
        }
        return $priority;
    }


    // public function getUserQuota($uid)
    // {
    //     $quota = exec("curl http://rankwatch.com/getuserquota?$uid; echo");

    //     if(empty($quota)) {
    //         $this->common->logErrorFatal('', "service:", "unable to get quota: $uid", print_r(error_get_last(), true));

    //         $result = array(
    //             'response' => 'Unable to retrive user quota',
    //             'response_code' => 101
    //         );

    //         echo json_encode($result);
    //         exit();
    //     } else {
    //         return $quota;
    //     }
    // }


    // public function updateUserQuota($uid, $new_quota)
    // {

    // }

    // public function getProjPages($pid)
    // {
    //     $qry = "SELECT pages_to_crawl FROM projects WHERE id_hash='".$pid."'";

    //     if($chk = mysqli_query($this->con_dist, $qry)) {
    //         if(mysqli_num_rows($chk)) {
    //             $res = mysqli_fetch_assoc($chk);
    //             if(empty($res['pages_to_crawl']) && $res['pages_to_crawl'] !== 0) {
    //                 $this->common->logErrorFatal('', "service:", "mysql:$qry", print_r(error_get_last(), true));
    //                 $msg = "Unable to get PID pages";
    //                 $err = 1;
    //             } else {
    //                 return $res['pages_to_crawl'];
    //             }
    //         } else {
    //             $this->common->logErrorFatal('', "service:", "no entry found for $pid", print_r(error_get_last(), true));
    //             $msg = "No entry found for PID";
    //             $err = 1;
    //         }
    //     } else {
    //         $this->common->logErrorFatal('', "service:", mysqli_error($this->con_dist), print_r(error_get_last(), true));
    //         $msg = "Internal Error";
    //         $err = 1;
    //     }

    //     if(!empty($err)) {
    //         $result = array(
    //             'response' => $msg,
    //             'response_code' => 101
    //         );

    //         echo json_encode($result);
    //         exit();
    //     }
    // }


    public function getUserProjCount($uid, $pid)
    {
        $qry = "SELECT COUNT(id) AS total_pages FROM projects WHERE user_id=$uid AND id != $pid AND expired=0";

        if($chk = mysqli_query($this->con_dist, $qry)) {
            if(mysqli_num_rows($chk)) {
                $res = mysqli_fetch_assoc($chk);
                if(empty($res['total_pages']) && $res['total_pages'] != 0) {
                    $this->common->logErrorFatal('', "service:", "mysql:$qry", print_r(error_get_last(), true));
                    $msg = "Unable to get PID pages";
                    $err = 1;
                } else {
                    return $res['total_pages'];
                }
            } else {
                $this->common->logErrorFatal('', "service:", "no entry found for $uid", print_r(error_get_last(), true));
                $msg = "No entry found for PID";
                $err = 1;
            }
        } else {
            $this->common->logErrorFatal('', "service:", mysqli_error($this->con_dist), print_r(error_get_last(), true));
            $msg = "Internal Error";
            $err = 1;
        }

        if(!empty($err)) {
            $result = array(
                'response' => $msg,
                'response_code' => 101
            );

            echo json_encode($result);
            exit();
        }
    }


    public function getAllProjPages($uid, $pid, $new="")
    {
        if(!empty($new)) {
            $qry = "SELECT SUM(pages_to_crawl) AS total_pages FROM projects WHERE user_id=$uid AND expired=0";
        } else {
            $qry = "SELECT SUM(pages_to_crawl) AS total_pages FROM projects WHERE user_id=$uid AND id != $pid AND expired=0";
        }

        if($chk = mysqli_query($this->con_dist, $qry)) {
            if(mysqli_num_rows($chk)) {
                $res = mysqli_fetch_assoc($chk);
                if(empty($res['total_pages']) && $res['total_pages'] != 0) {
                    $this->common->logErrorFatal('', "service:", "mysql:$qry", print_r(error_get_last(), true));
                    $msg = "Unable to get PID pages";
                    $err = 1;
                } else {
                    return $res['total_pages'];
                }
            } else {
                $this->common->logErrorFatal('', "service:", "no entry found for $uid", print_r(error_get_last(), true));
                $msg = "No entry found for PID";
                $err = 1;
            }
        } else {
            $this->common->logErrorFatal('', "service:", mysqli_error($this->con_dist), print_r(error_get_last(), true));
            $msg = "Internal Error";
            $err = 1;
        }

        if(!empty($err)) {
            $result = array(
                'response' => $msg,
                'response_code' => 101
            );

            echo json_encode($result);
            exit();
        }
    }

    // public function chkQuotaLimit($u_id, $p_id, $new_pages, $p_class) {
    //     if($p_class == 1) {
    //         $add_pages = $new_pages;
    //     }
    //     elseif($p_class == 2) {
    //         $old_pages = $this->getProjPages($p_id);
    //         $add_pages = $new_pages - $old_pages;
    //     }

    //     $total_old_pages = $this->getAllProjPages($u_id);

    //     $new_quota = $total_old_pages + $add_pages;
    //     $current_quota = $this->getUserQuota($u_id);

    //     if($new_quota > $current_quota) {
    //         $result = array(
    //             'response' => "Project pages exceed Quota limit",
    //             'response_code' => 101
    //         );

    //         echo json_encode($result);
    //         exit();
    //     }
    //     elseif($new_quota <= $current_quota) {
    //         return true;
    //     }

    // }


    public function getPerUrlGraph($data)
    {
        $url = $data['url'];
        $graph_data = array();
        $desc_arr = array(
            'mal' => array('Missing Image Alt', 'have missing ALT in images', 'eimg.mal'),
            'mkw' => array('Missing Meta Keyword', 'have meta keywords missing', 'emet.mkw'),
            'lkw' => array('Long Meta Keyword', 'have too long meta keyword', 'emet.lkw'),
            'skw' => array('Short Meta Keyword', 'have too short meta keyword', 'emet.skw'),
            'mds' => array('Missing Meta Description', 'have meta description missing', 'emet.mds'),
            'lds' => array('Long Meta Description', 'have too long meta description', 'emet.lds'),
            'sds' => array('Short Meta Description', 'have too short meta keyword', 'emet.sds'),
            'thn' => array('Thin Content', 'have thin content', 'epage.thn'),
            'low' => array('Text to Html Ratio', 'have low text to html ratio', 'etth.low'),
            'dti' => array('Duplicate Title', 'have duplicate title in them', 'etit.dti'),
            'dkw' => array('Duplicate Meta Keywords', 'have duplicate meta keywords in them', 'emet.dkw'),
            'dde' => array('Duplicate Meta Description', 'have duplicate meta description in them', 'emet.dde'),
            'tu' => array('Duplicate Content', 'have duplicate content in them', 'tu'),
            'lt' => array('Long Title', 'have too long title', 'etit.lt'),
            'sti' => array('Short Title', 'have too short title', 'etit.sti'),
            'eti' => array('Empty/Missing Title', 'have empty title', 'etit.eti'),
            'mrf' => array('Meta Refresh', 'have meta refresh', 'emet.mrf'),
            'mre' => array('Meta Redirect', 'have meta redirect', 'emet.mre'),
            'mnf' => array('Meta Nofollow', 'have meta nofollow', 'emet.mnf'),
            'mlg' => array('Missing Language Declaration', 'have language declaration missing', 'mlg'),
            'mst' => array('Too Many Stylesheets', 'have too many stylesheets included', 'esht.mst'),
            'mh1' => array('Missing H1 Tag', 'have missing h1 tag', 'eh1s.mh1'),
            'eh1' => array('Too Many H1 Tags', 'have too many h1 tags', 'eh1s.eh1'),
            'mh2' => array('Missing H2 Tag', 'have missing h2 tag', 'eh2s.mh2'),
            'eh2' => array('Too Many H2 Tags', 'have too many h2 tags', 'eh2s.eh2'),
            'lu' => array('Long URL', 'have too long URL', 'eurl.lu'),
            'spu' => array('Special Characters in URL', 'have special characters in URL', 'eurl.spu'),
            'unu' => array('Underscore Characters in URL', 'have underscores in URL', 'eurl.unu'),
            'upu' => array('UpperCase Characters in URL', 'have uppercase characters in URL', 'eurl.upu'),
            'frm' => array('Using Iframes', 'have iframes in content', 'efrm.frm'),
            'fls' => array('Using Flash', 'have flash in content', 'efls.fls'),
            'ent' => array('Too Many Internal Links', 'have too many internal links', 'eln.ent'),
            'eex' => array('Too Many External Links', 'have too many external links', 'eln.eex'),
            'nfc' => array('NoFollow Links', 'have nofollow links', 'eln.nfc'),
            'eim' => array('Too many images', 'have too many images', 'eimg.eim'),
            // 'esp' => array('Too many spelling mistakes', 'have too many spelling mistakes', 'espl.esp'),
            't' => array('Slow Page Speed', 'have slow response time', 't'),
            'big' => array('Page Size too Big', 'have big page size', 'epage.big'),
            'cms' => array('Missing Characterset Declaration', 'have characterset declaration missing', 'emet.cms'),
            'dms' => array('Missing Doctype Declaration', 'have doctype declaration missing', 'edoc.dms')
        );
        foreach ($desc_arr as $eckey => $ecvalue) {
            $err_col_arr[$ecvalue[2]] = 1;
        }


        $qry = "SELECT id,domain,dup_running FROM projects WHERE id_hash='".$data['p_id']."'";
        if($chk = mysqli_query($this->con_dist, $qry)) {
            if(mysqli_num_rows($chk)) {
                $res = mysqli_fetch_assoc($chk);
                $pid = $res['id'];

                if($res['dup_running'] == 1) {
                    $qry = "SELECT id FROM projects WHERE domain='".$res['domain']."' AND dup_running=0";
                    $chk = mysqli_query($this->con_dist, $qry);
                    $res = mysqli_fetch_assoc($chk);
                    $pid = (int)$res['id'];
                }

            }
        } else {
            $this->common->logErrorFatal('', "mysql:$qry", mysqli_error($this->con_dist), print_r(error_get_last(), true));
        }

        $mongo = $this->dbc->conMongo();
        if (!is_object($mongo)) {
            $this->common->logErrorFatal('', 'mongo:', 'mongo connection object is unusable'.print_r($mongo, true), print_r(error_get_last(), true));
            return;
        }

        $u_his = $mongo->auditor->find(
            array('pi' => (int)$pid, 'u' => $url),
            $err_col_arr
        );
        $u_his->limit(4);

        while($url_errs = $u_his->getNext()) {
            $e_err_arr[] = $url_errs;
        }


        foreach ($e_err_arr as $erval) {
            $created_on = $erval['_id']->getTimestamp();
            $created_on = date('Y-m-d', $created_on);
            unset($erval['_id']);

            if(!array_key_exists($created_on, $date_arr)) {
                $date_arr[$created_on] = 1;
                $each_url_arr = array();

                foreach ($erval as $eskey => $esvalue) {
                    if(is_array($esvalue))
                    {
                        foreach ($esvalue as $esk => $esv) {
                            if(!empty($esvalue[$esk]) && $esvalue[$esk] >= 1) {
                                $page_errors[$esk] = 1;
                            }
                        }
                    }
                    elseif(!empty($erval[$eskey]) && $erval[$eskey] >= 1) {
                        $page_errors[$eskey] = 1;
                    }
                }

                if($erval['t'] > T_SPEED_SLOW)
                {
                    if(empty($page_errors['t']))
                        $page_errors['t'] = 0;

                    $page_errors['t'] += 1;
                }
                $graph_data[] = $page_errors;
                $graph_date[] = date('Y-m-d', $created_on);
            }
        }

        $result = array(
            'response' => "setting data",
            'data' => $graph_data,
            'date' => $graph_date,
            'response_code' => 100
        );
        return json_encode($result);

    }


    public function getPerErrorGraph($data)
    {
        $error = $data['error'];
        $graph_data = array();
        $graph_date = array();
        $desc_arr = array(
            'mal' => array('Missing Image Alt', 'have missing ALT in images', 'eimg.mal'),
            'mkw' => array('Missing Meta Keyword', 'have meta keywords missing', 'emet.mkw'),
            'lkw' => array('Long Meta Keyword', 'have too long meta keyword', 'emet.lkw'),
            'skw' => array('Short Meta Keyword', 'have too short meta keyword', 'emet.skw'),
            'mds' => array('Missing Meta Description', 'have meta description missing', 'emet.mds'),
            'lds' => array('Long Meta Description', 'have too long meta description', 'emet.lds'),
            'sds' => array('Short Meta Description', 'have too short meta keyword', 'emet.sds'),
            'thn' => array('Thin Content', 'have thin content', 'epage.thn'),
            'low' => array('Text to Html Ratio', 'have low text to html ratio', 'etth.low'),
            'dti' => array('Duplicate Title', 'have duplicate title in them', 'etit.dti'),
            'dkw' => array('Duplicate Meta Keywords', 'have duplicate meta keywords in them', 'emet.dkw'),
            'dde' => array('Duplicate Meta Description', 'have duplicate meta description in them', 'emet.dde'),
            'tu' => array('Duplicate Content', 'have duplicate content in them', 'tu'),
            'lt' => array('Long Title', 'have too long title', 'etit.lt'),
            'sti' => array('Short Title', 'have too short title', 'etit.sti'),
            'eti' => array('Empty/Missing Title', 'have empty title', 'etit.eti'),
            'mrf' => array('Meta Refresh', 'have meta refresh', 'emet.mrf'),
            'mre' => array('Meta Redirect', 'have meta redirect', 'emet.mre'),
            'mnf' => array('Meta Nofollow', 'have meta nofollow', 'emet.mnf'),
            'mlg' => array('Missing Language Declaration', 'have language declaration missing', 'mlg'),
            'mst' => array('Too Many Stylesheets', 'have too many stylesheets included', 'esht.mst'),
            'mh1' => array('Missing H1 Tag', 'have missing h1 tag', 'eh1s.mh1'),
            'eh1' => array('Too Many H1 Tags', 'have too many h1 tags', 'eh1s.eh1'),
            'mh2' => array('Missing H2 Tag', 'have missing h2 tag', 'eh2s.mh2'),
            'eh2' => array('Too Many H2 Tags', 'have too many h2 tags', 'eh2s.eh2'),
            'lu' => array('Long URL', 'have too long URL', 'eurl.lu'),
            'spu' => array('Special Characters in URL', 'have special characters in URL', 'eurl.spu'),
            'unu' => array('Underscore Characters in URL', 'have underscores in URL', 'eurl.unu'),
            'upu' => array('UpperCase Characters in URL', 'have uppercase characters in URL', 'eurl.upu'),
            'frm' => array('Using Iframes', 'have iframes in content', 'efrm.frm'),
            'fls' => array('Using Flash', 'have flash in content', 'efls.fls'),
            'ent' => array('Too Many Internal Links', 'have too many internal links', 'eln.ent'),
            'eex' => array('Too Many External Links', 'have too many external links', 'eln.eex'),
            'nfc' => array('NoFollow Links', 'have nofollow links', 'eln.nfc'),
            'eim' => array('Too many images', 'have too many images', 'eimg.eim'),
            // 'esp' => array('Too many spelling mistakes', 'have too many spelling mistakes', 'espl.esp'),
            't' => array('Slow Page Speed', 'have slow response time', 't'),
            'big' => array('Page Size too Big', 'have big page size', 'epage.big'),
            'cms' => array('Missing Characterset Declaration', 'have characterset declaration missing', 'emet.cms'),
            'dms' => array('Missing Doctype Declaration', 'have doctype declaration missing', 'edoc.dms')
        );


        $qry = "SELECT id,domain,dup_running FROM projects WHERE id_hash='".$data['p_id']."'";
        if($chk = mysqli_query($this->con_dist, $qry)) {
            if(mysqli_num_rows($chk)) {
                $res = mysqli_fetch_assoc($chk);
                $pid = $res['id'];

                if($res['dup_running'] == 1) {
                    $qry = "SELECT id FROM projects WHERE domain='".$res['domain']."' AND dup_running=0";
                    $chk = mysqli_query($this->con_dist, $qry);
                    $res = mysqli_fetch_assoc($chk);
                    $pid = (int)$res['id'];
                }

            }
        } else {
            $this->common->logErrorFatal('', "mysql:$qry", mysqli_error($this->con_dist), print_r(error_get_last(), true));
        }

        $qry2 = "SELECT $error,created_on FROM history WHERE p_id=$pid ORDER BY week ASC LIMIT 10";
        if($chk2 = mysqli_query($this->con_dist, $qry2)) {
            if(mysqli_num_rows($chk2)) {
                while($res2 = mysqli_fetch_array($chk2)) {
                    $w_date = date('Y-m-d', strtotime($res2[1]));
                    $graph_data[] = $res2[0];
                    $graph_date[] = $w_date;
                }
            }
        } else {
            $this->common->logErrorFatal('', "mysql:$qry2", mysqli_error($this->con_dist), print_r(error_get_last(), true));
        }

        $result = array(
            'response' => "graph data",
            'data' => $graph_data,
            'date' => $graph_date,
            'err_name' => $desc_arr[$error][0],
            'response_code' => 100
        );
        return json_encode($result);
    }


    /**
     * Service to create runnow in Website Auditor
     *
     * Accepts GET params only
     *
     * @param int user_id
     * @param int pages_to_crawl
     * @param string p_id
     * @param string project (domain_name)
     * @param int user_class (1=>new free, 2=>paid, 3=>free)
     * @param int project_class (1=>new, 2=>quota_increase, 3=>weekly)
     *
     * Conditions:
     *   Create Entry:
     *       if no entry for PID
     *
     *   Update Entry:
     *       if entry exists for PID
     *
     *   Errors:
     *       if unable to create or update entry
     *       if GET parameters empty
     *       if PID exists but is associated with other domain
     *       if PID exists but is associated with other user
     *       if PID exists and is currently processing
     *
     *   Success:
     *       if entry created/updated successfully
     *
     *   Return Value:
     *       JSON encoded array,
     *       VALUES:
     *            response: response message
     *            response_code: 100(success)/101(error)
     *
     */
    public function newProjRunnow($data)
    {
        $resp_code = 101;

        if(!isset($data['user_id']) || !isset($data['pages_to_crawl']) || !isset($data['p_id']) || !isset($data['project'])|| !isset($data['user_class'])|| !isset($data['project_class'])) {

            $this->common->logErrorFatal('', 'service:', print_r($data, true), print_r(error_get_last(), true));

            $result = array(
                'response' => 'Incomplete Data',
                'response_code' => $resp_code
            );

            return json_encode($result);
        }

        $user_id = $data['user_id'];
        $pages_to_crawl = (int)$data['pages_to_crawl'];
        $p_id = $data['p_id'];
        $domain = $data['project'];
        $p_class = $data['project_class'];
        $u_class = $data['user_class'];
        $user_quota = $data['user_quota'];

        $pid = $this->getIdFromHash($data['p_id']);

        $tot_used = $this->getAllProjPages($data['user_id'], $pid, 1);

        $priority =  $this->calculatePriority($u_class, $p_class);

        // if(($tot_used + $pages_to_crawl) > $user_quota && $tot_used < $user_quota) {
        //     $pages_to_crawl = $user_quota - $tot_used;
        //     $makeProj = "INSERT INTO projects(id_hash,user_id,domain,pages_to_crawl,runnow,runnow_priority) VALUES('".$p_id."',$user_id,'".$domain."',$pages_to_crawl,1,$priority)";
        // }
        if(($tot_used + $pages_to_crawl) <= $user_quota) {
            $pages_to_crawl = (int)$data['pages_to_crawl'];
            $makeProj = "INSERT INTO projects(id_hash,user_id,domain,pages_to_crawl,runnow,runnow_priority) VALUES('".$p_id."',$user_id,'".$domain."',$pages_to_crawl,1,$priority)";
        } else {
            $pages_to_crawl = 0;
            $makeProj = "INSERT INTO projects(id_hash,user_id,domain,pages_to_crawl,runnow,expired,crawled,scraped,processed,duplicated,history_created,dump_deleted,status,last_crawled_on) VALUES('".$p_id."',$user_id,'".$domain."',$pages_to_crawl,0,2,2,2,2,2,2,2,2,NOW())";
        }

        // check if entry already exists for this project
        if($existingPID = $this->chkProjExists($p_id)) {
            $msg = 'PID already exists, runnow can\'t be created for existing PID';
        } else {
            if (mysqli_query($this->con_dist, $makeProj)) {
                // $this->updateUserQuota($user_id, $new_quota);
                $msg = 'Runnow created successfully';
                $resp_code = 100;
            }
            else {
                $this->common->logErrorFatal('', 'mysql:', mysqli_error($this->con_dist), print_r(error_get_last(), true));
                $msg = 'Unable to create runnow';
            }
        }
        $result = array(
            'response' => $msg,
            'response_code' => $resp_code
        );
        return json_encode($result);
    }


    public function delProject($data)
    {
        $qry = "SELECT * FROM projects WHERE id_hash='".$data['p_id']."'";
        if($chk = mysqli_query($this->con_dist, $qry)) {
            $res = mysqli_fetch_assoc($chk);
            // if($res['crawled'] != 2 || $res['scraped'] != 2 || $res['processed'] != 2 || $res['duplicated'] != 2 || $res['history_created'] != 2) {
            //     $result = array(
            //         'response' => "Project is currently processing, please wait until it's finished.",
            //         'response_code' => 101
            //     );
            //     return json_encode($result);
            // }
            $pid = $res['id'];

            if(!empty($pid)) {
                $del = "DELETE FROM projects WHERE id=$pid";
                if(mysqli_query($this->con_dist, $del)) {
                    if($res['dup_running'] == 0) {
                        $this->delProjMongo($pid);
                        $this->chkDupProjs($pid, $res['domain']);
                    }

                    $result = array(
                        'response' => "Project successfully deleted.",
                        'response_code' => 100
                    );
                    return json_encode($result);
                } else {
                    $this->common->logErrorFatal('', 'mysql:', mysqli_error($this->con_dist), print_r(error_get_last(), true));
                }
            }
        } else {
            $this->common->logErrorFatal('', 'mysql:', mysqli_error($this->con_dist), print_r(error_get_last(), true));
        }
    }


    public function delProjMongo($pid)
    {
        $mongo = $this->dbc->conMongo();
        if (!is_object($mongo)) {
            $this->common->logErrorFatal('', 'mongo:', 'mongo connection object is unusable'.print_r($mongo, true), print_r(error_get_last(), true));
            return;
        }

        if(!empty($pid)) {
            $pid = (int)$pid;

            $mongo->auditor->remove(array('pi' => $pid));
            $mongo->auditor_nonindex->remove(array('pi' => $pid));
            $mongo->auditor_non200->remove(array('pi' => $pid));
            $mongo->auditor_failed->remove(array('pi' => $pid));
        }
    }


    public function chkDupProjs($pid, $domain)
    {
        $qry = "SELECT id,pages_to_crawl,dup_running,crawl_no FROM projects WHERE domain='".$domain."' AND id != '".$pid."' AND expired=0  ORDER BY pages_to_crawl DESC LIMIT 1";
        if($chk = mysqli_query($this->con_dist, $qry)) {
            if(mysqli_num_rows($chk)) {
                $res = mysqli_fetch_assoc($chk);

                if($res['dup_running'] == 1) {
                    $this->markNominee($res['id'], $domain);
                }
            }
        } else {
            $this->common->logErrorFatal('', 'mysql:', mysqli_error($this->con_dist), print_r(error_get_last(), true));
        }
    }


    public function markNominee($pid, $domain)
    {
        $qry = "UPDATE projects SET runnow=1,crawled=0,scraped=0,processed=0,duplicated=0,history_created=0,dump_deleted=0,status=0,last_crawled_on=NOW(),dup_running=0,last_crawled_on=NULL WHERE domain='".$domain."' AND id = $pid";
        if(mysqli_query($this->con_dist, $qry)) {
            $qry2 = "UPDATE projects SET crawled=2,scraped=2,processed=2,duplicated=2,history_created=2,dump_deleted=2,status=2,last_crawled_on=NOW(),dup_running=1 WHERE id != $pid AND domain='".$domain."'";
            if(!mysqli_query($this->con_dist, $qry2)) {
                $this->common->logErrorFatal('', 'mysql:', mysqli_error($this->con_dist), print_r(error_get_last(), true));
            }
        } else {
            $this->common->logErrorFatal('', 'mysql:', mysqli_error($this->con_dist), print_r(error_get_last(), true));
        }
    }
// update mongo pid and weekno


    /**
     * Service to create runnow in Website Auditor
     *
     * Accepts GET params only
     *
     * @param int user_id
     * @param int pages_to_crawl (new quota for project)
     * @param string p_id
     * @param string project (domain_name)
     * @param int user_class (1=>new free, 2=>paid, 3=>free)
     * @param int project_class (1=>new, 2=>quota_increase, 3=>weekly)
     *
     * Return Value:
     *   JSON encoded array,
     *   VALUES:
     *     response: response message
     *     response_code: 100(success)/101(error)
     */
    public function QuotaIncrease($data)
    {
        $resp_code = 101;

        if(!isset($data['user_id']) || !isset($data['pages_to_crawl']) || !isset($data['p_id']) || !isset($data['project'])|| !isset($data['user_class'])|| !isset($data['project_class'])) {

            $this->common->logErrorFatal('', 'service:', print_r($data, true), print_r(error_get_last(), true));

            $result = array(
                'response' => 'Incomplete Data',
                'response_code' => $resp_code
            );

            return json_encode($result);
        }

        $user_id = $data['user_id'];
        $pages_to_crawl = (int)$data['pages_to_crawl'];
        $p_id = $data['p_id'];
        $p_class = $data['project_class'];
        $u_class = $data['user_class'];

        // $new_quota = $this->chkQuotaLimit($user_id, $p_id, $pages_to_crawl, $p_class);

        $priority =  $this->calculatePriority($u_class, $p_class);

        $qry = "UPDATE projects SET runnow=1,runnow_priority=98,last_crawled_on=NOW(),pages_to_crawl=$pages_to_crawl,pages_crawled=$pages_to_crawl,expired=0 WHERE id_hash='".$p_id."' AND user_id=$user_id";

        if(mysqli_query($this->con_dist, $qry)) {
            if(mysqli_affected_rows($this->con_dist)) {
                $msg = "Quota changed successfully";
                $resp_code = 100;
            } else {
                $this->common->logErrorFatal('', 'service:no row affected in quota change', $qry.":".mysqli_error($this->con_dist), print_r(error_get_last(), true));

                $msg = "Unable to change Quota";
            }
        } else {
            $this->common->logErrorFatal('', "service:$qry", mysqli_error($this->con_dist), print_r(error_get_last(), true));

            $msg = "Unable to change Quota";
        }

        $result = array(
            'response' => $msg,
            'response_code' => $resp_code
        );
        return json_encode($result);
    }


    /**
     * Service to create runnow in Website Auditor
     *
     * Accepts GET params only
     *
     * @param int user_id
     * @param int pages_to_crawl (new quota for project)
     * @param string p_id
     * @param string project (domain_name)
     * @param int user_class (1=>new free, 2=>paid, 3=>free)
     * @param int project_class (1=>new, 2=>quota_increase, 3=>weekly)
     *
     * Return Value:
     *   JSON encoded array,
     *   VALUES:
     *     response: response message
     *     response_code: 100(success)/101(error)
     */
   public function QuotaDecrease($data)
    {
        $resp_code = 101;

        if(!isset($data['user_id']) || !isset($data['pages_to_crawl']) || !isset($data['p_id']) || !isset($data['project'])|| !isset($data['user_class'])|| !isset($data['project_class'])) {

            $this->common->logErrorFatal('', 'service:', print_r($data, true), print_r(error_get_last(), true));

            $result = array(
                'response' => 'Incomplete Data',
                'response_code' => $resp_code
            );

            return json_encode($result);
        }

        $user_id = $data['user_id'];
        $pages_to_crawl = (int)$data['pages_to_crawl'];
        $p_id = $data['p_id'];
        $p_class = $data['project_class'];
        $u_class = $data['user_class'];

        $priority =  $this->calculatePriority($u_class, $p_class);

        $qry = "UPDATE projects SET runnow_priority=99,pages_to_crawl=$pages_to_crawl,pages_crawled=$pages_to_crawl,expired=0 WHERE id_hash='".$p_id."' AND user_id=$user_id";

        if(mysqli_query($this->con_dist, $qry)) {
            if(mysqli_affected_rows($this->con_dist)) {
                $msg = "Quota changed successfully";
                $resp_code = 100;
            } else {
                $this->common->logErrorFatal('', 'service:no row affected in quota change', $qry.":".mysqli_error($this->con_dist), print_r(error_get_last(), true));

                $msg = "Unable to change Quota";
            }
        } else {
            $this->common->logErrorFatal('', "service:$qry", mysqli_error($this->con_dist), print_r(error_get_last(), true));

            $msg = "Unable to change Quota";
        }

        $result = array(
            'response' => $msg,
            'response_code' => $resp_code
        );
        return json_encode($result);
    }


    /**
     * Service to change project setting in Website Auditor
     *
     * Accepts GET params only
     *
     * @param string p_id
     * @param int setting (user_agent or follow_robot)
     * @param int setting_value (0=>off, 1=>on In case of follow_robot), (string in Case of user_agent)
     *
     * Return Value:
     *   JSON encoded array,
     *   VALUES:
     *     response: response message
     *     response_code: 100(success)/101(error)
     */
    public function setProjSetting($data)
    {
        $set_arr = array();

        $pid = $this->getIdFromHash($data['p_id']);

        if(!isset($data['p_id']) || !isset($data['robot_value']) || !isset($data['agent_value']) || !isset($pid)) {

            $this->common->logErrorFatal('', 'service:', print_r($data, true), print_r(error_get_last(), true));

            $result = array(
                'response' => 'Incomplete Data',
                'response_code' => 101
            );

            return json_encode($result);
        }

        $set_arr['follow_robot'] = $data['robot_value'];
        $set_arr['user_agent'] = rawurldecode($data['agent_value']);

        $new_setting = serialize($set_arr);

        if($s_data = $this->chkProjSetting($pid)) {
            $qry = "UPDATE proj_settings SET setting = '".$new_setting."', changed = 1, updated_on = NOW() WHERE p_id=$pid";
        } else {
            $qry = "INSERT INTO proj_settings(p_id,setting,changed,created_on,updated_on) VALUES($pid, '".$new_setting."', 1, NOW(), NOW())";
        }

        if(mysqli_query($this->con_dist, $qry)) {
            $result = array(
                'response' => "setting changed successfully",
                'response_code' => 100
            );
        } else {
            $this->common->logErrorFatal('', 'service:', mysqli_error($this->con_dist), print_r(error_get_last(), true));

            $result = array(
                'response' => "Internal Error",
                'response_code' => 101
            );
        }
        return json_encode($result);

    }


    /**
     * Service to change project setting in Website Auditor
     *
     * Accepts GET params only
     *
     * @param string p_id
     *
     * Return Value:
     *   JSON encoded array,
     *   VALUES:
     *     response: response message
     *     response_code: 100(success)/101(error)
     */
    public function getProjSetting($data)
    {
        $resp_code = 101;
        $data_arr = array();

        $pid = $this->getIdFromHash($data['p_id']);

        if(!isset($data['p_id']) || !isset($data['user_id']) || !isset($pid)) {

            $this->common->logErrorFatal('', 'service:', print_r($data, true), print_r(error_get_last(), true));

            $result = array(
                'response' => 'Incomplete Data',
                'response_code' => $resp_code
            );

            return json_encode($result);
        }

        $tot_used = $this->getAllProjPages($data['user_id'], $pid);
        $tot_proj = $this->getUserProjCount($data['user_id'], $pid);

        $u_data_arr['name'] = "total_quota";
        $u_data_arr['setting'] = $tot_used;
        $data_arr[] = $u_data_arr;

        $p_data_arr['name'] = "total_projects";
        $p_data_arr['setting'] = $tot_proj;
        $data_arr[] = $p_data_arr;

        $qry = "SELECT pages_to_crawl FROM projects WHERE id = $pid";
        if($chk = mysqli_query($this->con_dist, $qry)) {
            while($res = mysqli_fetch_assoc($chk)) {
                $p_data_arr['name'] = "proj_quota";
                $p_data_arr['setting'] = $res['pages_to_crawl'];
                $data_arr[] = $p_data_arr;
            }
        } else {
            $this->common->logErrorFatal('', "service:$qry", mysqli_error($this->con_dist), print_r(error_get_last(), true));
        }


        if($s_data = $this->chkProjSetting($pid)) {
            $s_arr = unserialize($s_data['setting']);
            foreach ($s_arr as $key => $value) {
                $s_f_arr['name'] = $key;
                $s_f_arr['setting'] = $value;
                $data_arr[] = $s_f_arr;
            }
        } else {
            $qry = "SELECT name,setting FROM settings WHERE name = 'user_agent' OR name = 'follow_robot'";
            if($chk = mysqli_query($this->con_dist, $qry)) {
                while($res = mysqli_fetch_assoc($chk)) {
                    $data_arr[] = $res;
                }
            } else {
                $this->common->logErrorFatal('', "service:$qry", mysqli_error($this->con_dist), print_r(error_get_last(), true));
            }
        }

        $result = array(
            'response' => "setting data",
            'settings' => $data_arr,
            'response_code' => 100
        );
        return json_encode($result);

    }


    public function getIdFromHash($id_hash)
    {
        $qry = "SELECT id FROM projects WHERE id_hash='".$id_hash."'";

        if($chk = mysqli_query($this->con_dist, $qry)) {
            if(mysqli_num_rows($chk)) {
                $row = mysqli_fetch_assoc($chk);
                return $row['id'];
            }
        } else {
            $this->common->logErrorFatal('', "service:$qry", mysqli_error($this->con_dist), print_r(error_get_last(), true));
        }
        return false;
    }


    public function chkProjSetting($pid)
    {
        $qry = "SELECT id,setting FROM proj_settings WHERE p_id=$pid";

        if($chk = mysqli_query($this->con_dist, $qry)) {
            if(mysqli_num_rows($chk)) {
                $row = mysqli_fetch_assoc($chk);
                return $row;
            }
        } else {
            $this->common->logErrorFatal('', "service:$qry", mysqli_error($this->con_dist), print_r(error_get_last(), true));
        }
        return false;
    }


    /**
     * Service to project servers in Website Auditor
     *
     * Accepts GET params only
     *
     * @param string p_id, first_stat, action=get_proj_servers
     *
     * Return Value:
     *   JSON encoded array,
     *   VALUES:
     *     response: response message
     *     response_code: 100(success)/101(error)
     */
    public function getProjServers($data)
    {
        $master = "";
        $slaves = array();
        $res = array();
        $pages = array();
        $new_runnow = 0;
        $res['new_runnow'] = 0;
        $time_diff = 10;
        $status = 0;

        $p_data = $this->getProjData($data['p_id']);

        $pid = (int)$p_data['id'];
        if($p_data['dup_running'] == 1) {
            $dqry = "SELECT id FROM projects WHERE domain='".$p_data['domain']."' AND dup_running=0";
            $dchk = mysqli_query($this->con_dist, $dqry);
            $dres = mysqli_fetch_assoc($dchk);
            $pid = (int)$dres['id'];
        }

        // $this->chkDupRunning($data['p_id']);

        $qry = "SELECT crawled,scraped,processed,master_id,runnow,pages_to_crawl,status,last_crawled_on,runnow_priority,crawl_no FROM projects WHERE id='".$pid."'";
        if($chk = mysqli_query($this->con_dist, $qry)) {
            if(mysqli_num_rows($chk)) {
                $row = mysqli_fetch_assoc($chk);
                $master = $row['master_id'];
                $runnow = $row['runnow'];
                $pages_to_crawl = $row['pages_to_crawl'];
                if(!empty($row['last_crawled_on'])) {
                    $time_diff = (abs(strtotime($row['last_crawled_on']) - strtotime(date('Y-m-d H:i:s'))) / 86400);
                    $time_diff_hour = $time_diff * 24;
                    $time_diff_min = $time_diff_hour * 60;
                    $time_diff_sec = $time_diff_min * 60;
                }

                if($row['runnow_priority'] == 98 && $runnow == 1 && $time_diff_min > 0 && $row['status'] == 2) {
                    $quota_runnow = 1;
                    $status = 0;
                }
                elseif($row['status'] == 2 && $time_diff < 7) {
                    $status = 1;
                }
                elseif($row['status'] != 2 && ($row['crawled'] != 2 || $row['scraped'] != 2 || $row['processed'] != 2)) {
                    if($row['crawl_no'] > 1 && $row['runnow'] == 0) {
                        $status = 1;
                    } else {
                        $status = 0;
                    }
                }
                elseif((($row['status'] == 2 && $time_diff > 7) || ($row['status'] != 2 && $row['last_crawled_on'] == "")) && $row['crawl_no'] == 1) {
                    $status = 0;
                    $new_runnow = 1;
                } else {
                    $status = 1;
                }
            }
        } else {
            $this->common->logErrorFatal('', "service:$qry", mysqli_error($this->con_dist), print_r(error_get_last(), true));
        }


        $qry = "SELECT id FROM servers WHERE type=1";
        if($chk = mysqli_query($this->con_dist, $qry)) {
            if(mysqli_num_rows($chk)) {
                while($row = mysqli_fetch_assoc($chk)) {
                    $slaves[] = $row['id'];
                }
            }
        } else {
            $this->common->logErrorFatal('', "service:$qry", mysqli_error($this->con_dist), print_r(error_get_last(), true));
        }

        if(empty($master)) {
            $master = "";
        }

        if(!isset($data['runnow_request'])) {
            $res['master'] = $master;
            $res['slaves'] = $slaves;
            $res['runnow'] = $runnow;
            $res['pages_to_crawl'] = $pages_to_crawl;
        }
        $time_remaining = abs((5*60) - $time_diff_sec);
        if(isset($quota_runnow) && $quota_runnow == 1 && ($time_remaining * 60) <= 5) {
            $res['quota_duration'] = $time_remaining;
        }
        $res['completed'] = $status;

        if($new_runnow == 1) {
            $qry = "UPDATE projects SET runnow=1,status=0 WHERE id=".$pid;
            if(!mysqli_query($this->con_dist, $qry)) {
                $this->common->logErrorFatal('', "service:$qry", mysqli_error($this->con_dist), print_r(error_get_last(), true));
            } else {
                $res['new_runnow'] = 1;

                $result = array(
                    'response' => "server details",
                    'servers' => $res,
                    'response_code' => 100
                );
                return json_encode($result);
            }
        }

        if($status == 1 && isset($data['first_stat']) && $data['first_stat'] == 1) {
            $mongo = $this->dbc->conMongo();
            if (!is_object($mongo)) {
                $this->common->logErrorFatal('', 'mongo:', 'mongo connection object is unusable'.print_r($mongo, true), print_r(error_get_last(), true));
            }
            $m_pages = $mongo->auditor->find(array('pi' => (int)$pid), array('_id' => 0, 'u' => 1));
            $m_pages->limit(100);
            while($pgs = $m_pages->getNext()) {
                $pages[] = $pgs['u'];
            }
            if(count($pages) < 100) {
                $m_pages = $mongo->auditor_non200->find(array('pi' => (int)$pid), array('_id' => 0, 'u' => 1));
                $m_pages->limit(100);
                while($pgs = $m_pages->getNext()) {
                    $pages[] = $pgs['u'];
                }
            }
            if(count($pages) < 100) {
                $m_pages = $mongo->auditor_nonindex->find(array('pi' => (int)$pid), array('_id' => 0, 'u' => 1));
                $m_pages->limit(100);
                while($pgs = $m_pages->getNext()) {
                    $pages[] = $pgs['u'];
                }
            }
            if(count($pages) < 100) {
                $m_pages = $mongo->auditor_failed->find(array('pi' => (int)$pid), array('_id' => 0, 'u' => 1));
                $m_pages->limit(100);
                while($pgs = $m_pages->getNext()) {
                    $pages[] = $pgs['u'];
                }
            }
            $res['pages'] = $pages;
        }

        $result = array(
            'response' => "server details",
            'servers' => $res,
            'response_code' => 100
        );
        return json_encode($result);
    }


    public function getSlaveIp($s_ids)
    {
        $s_arr = array();
        $qry = "SELECT id,ip FROM servers WHERE id IN ($s_ids)";
        if($chk = mysqli_query($this->con_dist, $qry)) {
            if(mysqli_num_rows($chk)) {
                while($row = mysqli_fetch_assoc($chk)) {
                    $slaves = array();
                    $slaves['id'] = $row['id'];
                    $slaves['ip'] = $row['ip'];
                    $s_arr[] = $slaves;
                }
            }
        } else {
            $this->common->logErrorFatal('', "service:$qry", mysqli_error($this->con_dist), print_r(error_get_last(), true));
        }
        return $s_arr;
    }


    public function getPythonData($runnow, $master_ip, $pid)
    {
        $args = "$pid $master_ip $runnow";
        $file = "progress_checker.py $args";
        $last_l = exec("python progress_checker.py $args", $out, $err);
        return json_decode($out[0]);
    }


    /**
     * Service to project progress in Website Auditor
     *
     * Accepts GET params only
     *
     * @param string p_id
     * @param string slave
     * @param string runnow
     * @param string master
     * @param int last_count
     * @param string action=get_proj_progress
     *
     * Return Value:
     *   JSON encoded array,
     *   VALUES:
     *     response: response message
     *     response_code: 100(success)/101(error)
     */
    public function getProjProgress($data)
    {
        $res = array();
        $masters = array();
        $unprocessed = 0;
        $res['completed'] = 0;

        $runnow = $data['runnow'];
        $c_total = $data['last_crawled_pages'];
        $c_processed = $data['last_analyzed_pages'];

        $data['slaves'] = explode(",", $data['slaves']);
        if(empty($data['master'])) {
            $qry = "SELECT id FROM servers WHERE type=0";
            if($chk = mysqli_query($this->con_dist, $qry)) {
                if(mysqli_num_rows($chk)) {
                    while($row = mysqli_fetch_assoc($chk)) {
                        $masters[] = $row['id'];
                    }
                }
            } else {
                $this->common->logErrorFatal('', "service:$qry", mysqli_error($this->con_dist), print_r(error_get_last(), true));
            }
            $data['master'] = implode(",", $masters);
        }

        $p_data = $this->getProjData($data['p_id']);

        $pid = (int)$p_data['id'];
        if($p_data['dup_running'] == 1) {
            $dqry = "SELECT id FROM projects WHERE domain='".$p_data['domain']."' AND dup_running=0";
            $dchk = mysqli_query($this->con_dist, $dqry);
            $dres = mysqli_fetch_assoc($dchk);
            $pid = (int)$dres['id'];
        }


        $s_str = implode(",", $data['slaves']).",".$data['master'];
        $slave_ips = $this->getSlaveIp($s_str);

        $py_data = 0;
        foreach ($slave_ips as $value) {
            if(in_array($value['id'], $masters)) {
                $py_data += $this->getPythonData($runnow, $value['ip'], $pid);
            }
        }
        $res['crawled'] = $py_data;

        $p_found = $this->getProgressUrl($slave_ips, $pid, $c_total, $c_processed);
        array_reverse($p_found);

        if(empty($p_found)) {
            $p_found = array();
        }
        $res['pages'] = $p_found;

        $qry = "SELECT master_id,runnow,pages_to_crawl,status,runnow,crawl_no FROM projects WHERE id='".$pid."'";
        if($chk = mysqli_query($this->con_dist, $qry)) {
            if(mysqli_num_rows($chk)) {
                $row = mysqli_fetch_assoc($chk);
                if($row['status'] == 2 && $row['runnow'] != 1) {
                    $status = 1;
                } elseif($row['crawl_no'] > 1 && $row['runnow'] == 0) {
                    $status = 1;
                } else {
                    $status = 0;
                }
            }
        } else {
            $this->common->logErrorFatal('', "service:$qry", mysqli_error($this->con_dist), print_r(error_get_last(), true));
        }


        if((empty($p_found) || $unprocessed < 2) && $status == 1) {
            $res['completed'] = 1;
        } else {
            $res['completed'] = 0;
        }

        $result = array(
            'response' => "currently processing",
            'progress' => $res,
            'response_code' => 100
        );
        return json_encode($result);
    }


    public function getProgressUrl($slave_ips, $pid, $c_total, $c_processed)
    {
        $pages = array();
        $processed = 0;
        foreach ($slave_ips as $value) {
            $con_temp = new mysqli($value['ip'], $this->temp_user, $this->temp_pass, $this->temp_db);
            if (mysqli_connect_error()) {
                $this->common->logErrorFatal('', "connection:", mysqli_connect_error(), print_r(error_get_last(), true));
            }

            $qry = "SELECT u,processed FROM temp WHERE pi=$pid ORDER BY id Desc";
            if($chk = mysqli_query($con_temp, $qry)) {
                if(mysqli_num_rows($chk)) {
                    while($row = mysqli_fetch_assoc($chk)){
                        if($row['processed'] == 1) {
                            $processed += 1;
                        }
                        $pages[] = $row;
                    }
                }
            } else {
                $this->common->logErrorFatal('', "service:$qry", mysqli_error($con_temp), print_r(error_get_last(), true));
            }

            // if(!empty($pages) && (count($pages) >= 100)) {
            //     break;
            // }
        }
        $mongo = $this->dbc->conMongo();
        if (!is_object($mongo)) {
            $this->common->logErrorFatal('', 'mongo:', 'mongo connection object is unusable'.print_r($mongo, true), print_r(error_get_last(), true));
        }
        // if(count($pages) < $c_total) {
            $m_pages = $mongo->auditor->find(array('pi' => (int)$pid), array('_id' => 0, 'u' => 1));
            // $m_pages->limit(100);
            while($pgs = $m_pages->getNext()) {
                $row = array();
                $row['u'] = $pgs['u'];
                $row['processed'] = 1;
                $pages[] = $row;
                $processed += 1;

                // if(!empty($pages) && (count($pages) >= 100)) {
                //     break;
                // }
            }
        // }
        // if(count($pages) < $c_total) {
            $m_pages = $mongo->auditor_non200->find(array('pi' => (int)$pid), array('_id' => 0, 'u' => 1));
            // $m_pages->limit(100);
            while($pgs = $m_pages->getNext()) {
                $row = array();
                $row['u'] = $pgs['u'];
                $row['processed'] = 1;
                $pages[] = $row;
                $processed += 1;
                // if(!empty($pages) && (count($pages) >= 100)) {
                //     break;
                // }
            }
        // }
        // if(count($pages) < $c_total) {
            $m_pages = $mongo->auditor_nonindex->find(array('pi' => (int)$pid), array('_id' => 0, 'u' => 1));
            // $m_pages->limit(100);
            while($pgs = $m_pages->getNext()) {
                $row = array();
                $row['u'] = $pgs['u'];
                $row['processed'] = 1;
                $pages[] = $row;
                $processed += 1;
                // if(!empty($pages) && (count($pages) >= 100)) {
                //     break;
                // }
            }
        // }
        // if(count($pages) < $c_total) {
            $m_pages = $mongo->auditor_failed->find(array('pi' => (int)$pid), array('_id' => 0, 'u' => 1));
            // $m_pages->limit(100);
            while($pgs = $m_pages->getNext()) {
                $row = array();
                $row['u'] = $pgs['u'];
                $row['processed'] = 1;
                $pages[] = $row;
                $processed += 1;
                // if(!empty($pages) && (count($pages) >= 100)) {
                //     break;
                // }
            }
        // }
        return $pages;
    }


    public function getProjData($id_hash)
    {
        $res = array();
        $qry = "SELECT * FROM projects WHERE id_hash='".$id_hash."'";
        if($chk = mysqli_query($this->con_dist, $qry)) {
            $res = mysqli_fetch_assoc($chk);
        } else {
            $this->common->logErrorFatal('', "service:$qry", mysqli_error($con_dist), print_r(error_get_last(), true));
        }
        return $res;
    }


    public function chkDupRunning($id_hash)
    {
        $qry = "SELECT id,dup_running,domain FROM projects WHERE id_hash='".$id_hash."'";
        if($chk = mysqli_query($this->con_dist, $qry)) {
            if(mysqli_num_rows($chk)) {
                $row = mysqli_fetch_assoc($chk);

                if($row['dup_running'] == 1) {
                    $qry2 = "SELECT id FROM projects WHERE domain='".$row['domain']."' AND dup_running=0";
                    if($chk2 = mysqli_query($this->con_dist, $qry2)) {
                        if(mysqli_num_rows($chk2)) {
                            $row2 = mysqli_fetch_assoc($chk2);
                            $send_pid = $row2['id'];
                        } else {
                            $this->convertToRunnow($row['id']);
                            $send_pid = $row['id'];
                        }
                    } else {
                        $this->common->logErrorFatal('', "service:$qry", mysqli_error($con_dist), print_r(error_get_last(), true));
                    }
                } else {
                    return $row['id'];
                }
            }
        } else {
            $this->common->logErrorFatal('', "service:$qry", mysqli_error($con_dist), print_r(error_get_last(), true));
        }
    }


    public function convertToRunnow($pid)
    {
        $qry = "UPDATE projects SET runnow=1,status=0 WHERE id=".$pid;
        if(!mysqli_query($this->con_dist, $qry)) {
            $this->common->logErrorFatal('', "service:$qry", mysqli_error($con_dist), print_r(error_get_last(), true));
        }
    }


    public function weeklyEmail($data)
    {
        $res_arr = array();

        $proj_data_arr = $this->chkProjData($data['user_id']);
        $chk_proj_data = $proj_data_arr[0];

        $mongo = $this->dbc->conMongo();
        if (!is_object($mongo)) {
            $this->common->logErrorFatal('', 'mongo:', 'mongo connection object is unusable'.print_r($mongo, true), print_r(error_get_last(), true));
            return;
        }

        foreach ($proj_data_arr as $pjkey => $pjvalue) {
            $pid = (int)$pjvalue['id'];
            $week = ((int)$pjvalue['crawl_no'] - 1);

            $totalScraped = $mongo->auditor->count(
                array('pi' => $pid, 'w' => $week)
            );

            $totalDups = $mongo->auditor->count(
                array('pi' => $pid, 'w' => $week, 'tu' => array('$gt' => 0))
            );

            $totalUnique = $totalScraped - $totalDups;

            $tCountNon200 = $mongo->auditor_non200->count(
                array('pi' => $pid, 'w' => $week)
            );

            $tCountNon200_ni = $mongo->auditor_nonindex->count(
                array('pi' => $pid, 'w' => $week)
            );

            $tCountFailed = $mongo->auditor_failed->count(
                array('pi' => $pid, 'w' => $week)
            );

            if($pjvalue['runnow_priority'] == 99) {
                $red_data = $this->getReducesData($totalUnique, $totalDups, $tCountNon200, $tCountNon200_ni, $tCountFailed, $pjvalue['pages_to_crawl']);

                if(!empty($red_data['unique'])) {
                    $totalUnique = $red_data['unique'];
                }
                if(!empty($red_data['duplicate'])) {
                    $totalDups = $red_data['duplicate'];
                }
                if(!empty($red_data['non200'])) {
                    $tCountNon200 = $red_data['non200'];
                }
                if(!empty($red_data['nonindex'])) {
                    $tCountNon200_ni = $red_data['nonindex'];
                }
                if(!empty($red_data['failed'])) {
                    $tCountFailed = $red_data['failed'];
                }
            }
            if($pjkey == 0) {
                $section = "main";
            } else {
                $section = "additional";
            }

            if($section == "main") {
                $err_major = 0;
                $err_medium = 0;
                $err_minor = 0;
                $e_err_arr = array();

                $desc_arr = array(
                    'mal' => array('Missing Image Alt', 'have missing ALT in images', 'eimg.mal'),
                    'mkw' => array('Missing Meta Keyword', 'have meta keywords missing', 'emet.mkw'),
                    'lkw' => array('Long Meta Keyword', 'have too long meta keyword', 'emet.lkw'),
                    'skw' => array('Short Meta Keyword', 'have too short meta keyword', 'emet.skw'),
                    'mds' => array('Missing Meta Description', 'have meta description missing', 'emet.mds'),
                    'lds' => array('Long Meta Description', 'have too long meta description', 'emet.lds'),
                    'sds' => array('Short Meta Description', 'have too short meta keyword', 'emet.sds'),
                    'thn' => array('Thin Content', 'have thin content', 'epage.thn'),
                    'low' => array('Text to Html Ratio', 'have low text to html ratio', 'etth.low'),
                    'dti' => array('Duplicate Title', 'have duplicate title in them', 'etit.dti'),
                    'dkw' => array('Duplicate Meta Keywords', 'have duplicate meta keywords in them', 'emet.dkw'),
                    'dde' => array('Duplicate Meta Description', 'have duplicate meta description in them', 'emet.dde'),
                    'tu' => array('Duplicate Content', 'have duplicate content in them', 'tu'),
                    'lt' => array('Long Title', 'have too long title', 'etit.lt'),
                    'sti' => array('Short Title', 'have too short title', 'etit.sti'),
                    'eti' => array('Empty/Missing Title', 'have empty title', 'etit.eti'),
                    'mrf' => array('Meta Refresh', 'have meta refresh', 'emet.mrf'),
                    'mre' => array('Meta Redirect', 'have meta redirect', 'emet.mre'),
                    'mnf' => array('Meta Nofollow', 'have meta nofollow', 'emet.mnf'),
                    'mlg' => array('Missing Language Declaration', 'have language declaration missing', 'mlg'),
                    'mst' => array('Too Many Stylesheets', 'have too many stylesheets included', 'esht.mst'),
                    'mh1' => array('Missing H1 Tag', 'have missing h1 tag', 'eh1s.mh1'),
                    'eh1' => array('Too Many H1 Tags', 'have too many h1 tags', 'eh1s.eh1'),
                    'mh2' => array('Missing H2 Tag', 'have missing h2 tag', 'eh2s.mh2'),
                    'eh2' => array('Too Many H2 Tags', 'have too many h2 tags', 'eh2s.eh2'),
                    'lu' => array('Long URL', 'have too long URL', 'eurl.lu'),
                    'spu' => array('Special Characters in URL', 'have special characters in URL', 'eurl.spu'),
                    'unu' => array('Underscore Characters in URL', 'have underscores in URL', 'eurl.unu'),
                    'upu' => array('UpperCase Characters in URL', 'have uppercase characters in URL', 'eurl.upu'),
                    'frm' => array('Using Iframes', 'have iframes in content', 'efrm.frm'),
                    'fls' => array('Using Flash', 'have flash in content', 'efls.fls'),
                    'ent' => array('Too Many Internal Links', 'have too many internal links', 'eln.ent'),
                    'eex' => array('Too Many External Links', 'have too many external links', 'eln.eex'),
                    'nfc' => array('NoFollow Links', 'have nofollow links', 'eln.nfc'),
                    'eim' => array('Too many images', 'have too many images', 'eimg.eim'),
                    't' => array('Slow Page Speed', 'have slow response time', 't'),
                    'big' => array('Page Size too Big', 'have big page size', 'epage.big'),
                    'cms' => array('Missing Characterset Declaration', 'have characterset declaration missing', 'emet.cms'),
                    'dms' => array('Missing Doctype Declaration', 'have doctype declaration missing', 'edoc.dms')
                );

                foreach ($desc_arr as $eckey => $ecvalue) {
                    $err_col_arr[$ecvalue[2]] = 1;
                }


                $processed_arr = array(
                );

                $final_arr = array_merge($desc_arr, $processed_arr);

                $err_col_arr['_id'] = 0; $err_col_arr['pos'] = 1;

                $each_errors = $mongo->auditor->find(
                    array('pi' => $pid, 'w' => $week),
                    $err_col_arr
                );
                $each_errors->limit((int)$pjvalue['pages_to_crawl']);

                $page_errors = array();

                while($e_error = $each_errors->getNext())
                {
                    $e_err_arr[] = $e_error;
                }
                foreach ($e_err_arr as $erval) {
                    foreach ($erval as $eskey => $esvalue) {
                        if($eskey !== "u" && $eskey !== "pos" && $eskey !== 't') {
                            if(is_array($esvalue)) {
                                foreach ($esvalue as $esk => $esv) {
                                    if(!empty($esvalue[$esk]) && $esvalue[$esk] >= 1 && $esk !== 'tit') {
                                        if(empty($page_errors[$esk])) {
                                            $page_errors[$esk] = 0;
                                        }
                                        $page_errors[$esk] += 1;
                                    }
                                }
                            }
                            elseif(!empty($erval[$eskey]) && $erval[$eskey] >= 1 && $eskey !== 'tit') {
                                if(empty($page_errors[$eskey])) {
                                    $page_errors[$eskey] = 0;
                                }
                                $page_errors[$eskey] += 1;
                            }
                        }
                    }

                    if(!empty($erval['t']) && $erval['t'] > T_SPEED_SLOW) {
                        if(empty($page_errors['t'])) {
                            $page_errors['t'] = 0;
                        }
                        $page_errors['t'] += 1;
                    }
                }
                foreach ($page_errors as $pkey => $pvalue) {
                    if(strstr(T_ERR_MAJOR, $pkey)) {
                        $err_major += 1;
                    }
                    elseif(strstr(T_ERR_MEDIUM, $pkey)) {
                        $err_medium += 1;
                    }
                    elseif(strstr(T_ERR_MINOR, $pkey)) {
                        $err_minor += 1;
                    }
                }
                $res_arr[$section][$pjvalue['id_hash']]['current'] = array('t_unique' => $totalUnique, 't_duplicate' => $totalDups, 't_non200' => $tCountNon200, 't_nonindex' => $tCountNon200_ni, 't_failed' => $tCountFailed, 't_major' => $err_major, 't_medium' => $err_medium, 't_minor' => $err_minor, 'domain' => $pjvalue['domain']);
            } else {
                $res_arr[$section][$pjvalue['id_hash']]['current'] = array('t_unique' => $totalUnique, 't_duplicate' => $totalDups, 't_non200' => $tCountNon200, 't_nonindex' => $tCountNon200_ni, 't_failed' => $tCountFailed, 'domain' => $pjvalue['domain']);
            }

            $res_arr[$section][$pjvalue['id_hash']]['history'] = $this->getHistoryData($pid, $week, $pjvalue, $mongo);
        }


        $result = array(
            'response' => "Weekly Email Data",
            'progress' => $res_arr,
            'response_code' => 100
        );
        return json_encode($result);
    }


    public function chkProjData($user_id)
    {
        $proj_arr = array();

        // $qry = "SELECT id,domain,runnow_priority,pages_to_crawl,crawl_no,dup_running FROM projects WHERE id_hash='".$id_hash."'";
        // if($chk = mysqli_query($this->con_dist, $qry)) {
        //     $res = mysqli_fetch_assoc($chk);
        //     $final_arr['orig'] = $res;

        //     if($res['dup_running'] == 1) {
        //         $qry = "SELECT id,domain,runnow_priority,pages_to_crawl,crawl_no FROM projects WHERE domain='".$res['domain']."' AND dup_running=0";
        //         if($chk = mysqli_query($this->con_dist, $qry)) {
        //             $res = mysqli_fetch_assoc($chk);
        //             $final_arr['data'] = $res;
        //         } else {
        //             $this->common->logErrorFatal('', "service:$qry", mysqli_error($con_dist), print_r(error_get_last(), true));
        //         }
        //     } else {
        //         $final_arr['data'] = $res;
        //     }
        // } else {
        //     $this->common->logErrorFatal('', "service:$qry", mysqli_error($con_dist), print_r(error_get_last(), true));
        // }
        // return $final_arr;

        $qry = "SELECT id,id_hash,domain,runnow_priority,pages_to_crawl,crawl_no FROM projects WHERE user_id=$user_id AND dup_running=0 AND expired=0 ORDER BY pages_to_crawl DESC LIMIT 10";
        if($chk = mysqli_query($this->con_dist, $qry)) {
            while($res = mysqli_fetch_assoc($chk)) {
                $proj_arr[] = $res;
            }
        } else {
            $this->common->logErrorFatal('', "service:$qry", mysqli_error($con_dist), print_r(error_get_last(), true));
        }
        return $proj_arr;

    }


    function getHistoryData($pid, $week, $chk_proj_data, $mongo)
    {
        $week = $week - 1;
        $pid = (int)$pid;

        $totalScraped = $mongo->auditor->count(
            array('pi' => $pid, 'w' => $week)
        );

        $totalDups = $mongo->auditor->count(
            array('pi' => $pid, 'w' => $week, 'tu' => array('$gt' => 0))
        );

        $totalUnique = $totalScraped - $totalDups;

        $tCountNon200 = $mongo->auditor_non200->count(
            array('pi' => $pid, 'w' => $week)
        );

        $tCountNon200_ni = $mongo->auditor_nonindex->count(
            array('pi' => $pid, 'w' => $week)
        );

        $tCountFailed = $mongo->auditor_failed->count(
            array('pi' => $pid, 'w' => $week)
        );

        if($chk_proj_data['runnow_priority'] == 99) {
            $red_data = $this->getReducesData($totalUnique, $totalDups, $tCountNon200, $tCountNon200_ni, $tCountFailed, $chk_proj_data['pages_to_crawl']);

            if(!empty($red_data['unique'])) {
                $totalUnique = $red_data['unique'];
            }
            if(!empty($red_data['duplicate'])) {
                $totalUnique = $red_data['duplicate'];
            }
            if(!empty($red_data['non200'])) {
                $tCountNon200 = $red_data['non200'];
            }
            if(!empty($red_data['nonindex'])) {
                $tCountNon200_ni = $red_data['nonindex'];
            }
            if(!empty($red_data['failed'])) {
                $tCountFailed = $red_data['failed'];
            }
        }
        $page_errors['t_unique'] = $totalUnique;
        $page_errors['t_duplicate'] = $totalDups;
        $page_errors['t_non200'] = $tCountNon200;
        $page_errors['t_nonindex'] = $tCountNon200_ni;
        $page_errors['t_failed'] = $tCountFailed;
        // $page_errors['t_crawled'] = $totalUnique + $totalDups + $tCountNon200 + $tCountNon200_ni + $tCountFailed;

        return $page_errors;
    }


    function getReducesData($totalUnique, $totalDups, $tCountNon200, $tCountNon200_ni, $tCountFailed, $total_pages)
    {
        $unique_count = 0;
        $dup_count = 0;
        $non200_count = 0;
        $nonindex_count = 0;
        $failed_count = 0;
        $pages_found = 0;
        while($pages_found <= $total_pages) {

            if($totalUnique > 0) {
                $pages_found++;
                $unique_count++;

                $totalUnique--;
                $total_pages--;
            }

            if($totalDups > 0) {
                $pages_found++;
                $dup_count++;

                $totalDups--;
                $total_pages--;
            }

            if($tCountNon200 > 0) {
                $pages_found++;
                $non200_count++;

                $tCountNon200--;
                $total_pages--;
            }

            if($tCountNon200_ni > 0) {
                $pages_found++;
                $nonindex_count++;

                $tCountNon200_ni--;
                $total_pages--;
            }

            if($tCountFailed > 0) {
                $pages_found++;
                $failed_count++;

                $tCountFailed--;
                $total_pages--;
            }
        }
        return array("unique"=>$unique_count,"duplicate"=>$dup_count,"non200"=>$non200_count,"nonindex"=>$nonindex_count,"failed"=>$failed_count);
    }


}