
<?php
    error_reporting(E_ALL);
    ini_set('display_errors', 'On');

    ini_set('memory_limit', '1024M');
    set_time_limit(5000);
    // if website form submitted for checking duplicate content
if (!empty($_GET['project'])) {
    require_once "conf.php"; //common configuration constants and error function defined

    $stime = time();

    $page_info = array();
    $err_major = array();
    $err_medium = array();
    $err_minor = array();
    $grade_arr = array();
    $page_errors = array();
    $url_error = array();
    $url_grade = array();
    $url_pos = array();
    $u_agents = array();
    $history_errors = array();

    $tCountNon200 = 0;
    $tCountNon200_ni = 0;

    $proj_data = getProjData($my_dist_host, $my_dist_user, $my_dist_pass, $my_dist_db);
    $proj_data_orig = getOrigProjData($my_dist_host, $my_dist_user, $my_dist_pass, $my_dist_db);
    $week = (int)$proj_data['crawl_no'] - 1;
    $last_checked_on = $proj_data['last_crawled_on'];
    $user_id = $proj_data['user_id'];

    $total_quota_used = getAllProjPages($proj_data_orig['user_id'], $proj_data_orig['id'], $my_dist_host, $my_dist_user, $my_dist_pass, $my_dist_db);

    $totalScraped = $mongoDB->auditor->count(
        array('pi' => P_ID, 'w' => $week)
    );

    $tCountNon200 = $mongoDB->auditor_non200->count(
        array('pi' => P_ID, 'w' => $week)
    );

    $tCountNon200_ni = $mongoDB->auditor_nonindex->count(
        array('pi' => P_ID, 'w' => $week)
    );

    $tCountFailed = $mongoDB->auditor_failed->count(
        array('pi' => P_ID, 'w' => $week)
    );

    if($proj_data['runnow_priority'] == 99) {
        $red_data = getReducesData($totalScraped, $tCountNon200, $tCountNon200_ni, $tCountFailed, $proj_data_orig['pages_to_crawl']);

        if(!empty($red_data['unique'])) {
            $totalScraped = $red_data['unique'];
        }
        if(!empty($red_data['non200'])) {
            $tCountNon200 = $red_data['non200'];
        }
        if(!empty($red_data['nonindex'])) {
            $tCountNon200_ni = $red_data['nonindex'];
        }
        if(!empty($red_data['failed'])) {
            $tCountFailed = $red_data['failed'];
        }
    }


    $page_errors = array();
    $e_err_arr = array();
    $robot_disallow = 0;

    // $robot_content = file_get_contents('http://'.URL_TO_CRAWL."/robots.txt");
    // preg_match_all('/disallow(:?)(.*?)\n/i', $robot_content, $m);

    $err_col_arr['_id'] = 0; $err_col_arr['u'] = $err_col_arr['pos'] = $err_col_arr['t'] = $err_col_arr['etit.tit'] = 1;

    $each_errors = $mongoDB->auditor->find(
        array('pi' => P_ID, 'w' => $week),
        $err_col_arr
    );
    $each_errors->limit((int)$totalScraped);

    $url_arr = array();

    while($e_error = $each_errors->getNext())
    {
        $e_err_arr[] = $e_error;
    }

    $totalScraped = $totalScraped + $tCountNon200 + $tCountNon200_ni + $tCountFailed;

    if(empty($totalScraped))
    {
        $result = json_encode(
            array(
                'response_code' => 101,
                'response_message' => "No Data Found for Project ". URL_TO_CRAWL
            )
        );
        echo $result; exit;
    }


    foreach ($e_err_arr as $erval) {
        $each_url_arr = array();
        foreach ($erval as $eskey => $esvalue) {

            if($eskey !== "u" && $eskey !== "pos" && $eskey !== 't')
            {
                if(is_array($esvalue))
                {
                    foreach ($esvalue as $esk => $esv) {
                        if(!empty($esvalue[$esk]) && $esvalue[$esk] >= 1 && $esk !== 'tit')
                        {
                            if(empty($page_errors[$esk]))
                                $page_errors[$esk] = 0;
                            $page_errors[$esk] += 1;
                            $each_url_arr[$esk] = 1;
                        }
                    }
                }
                elseif(!empty($erval[$eskey]) && $erval[$eskey] >= 1 && $eskey !== 'tit')
                {
                    if(empty($page_errors[$eskey]))
                        $page_errors[$eskey] = 0;
                    $page_errors[$eskey] += 1;
                    $each_url_arr[$eskey] = 1;
                }
            }

        }

        if($erval['t'] > T_SPEED_SLOW)
        {
            if(empty($page_errors['t']))
                $page_errors['t'] = 0;

            $page_errors['t'] += 1;
        }

        $erval['etit']['tit'] = empty($erval['etit']['tit']) ? "" : $erval['etit']['tit'];

        $erval['u'] = rawurlencode($erval['u']);

        $url_error[$erval['u']][$erval['etit']['tit']] = $each_url_arr;
        $url_grade[$erval['u']] = calculateGrade($each_url_arr);
        $url_pos[$erval['u']] = $erval['pos'];

        $grade_arr = array_count_values($url_grade);

        if(empty($grade_arr['A'])) $grade_arr['A'] = 0;
        if(empty($grade_arr['B'])) $grade_arr['B'] = 0;
        if(empty($grade_arr['C'])) $grade_arr['C'] = 0;
        if(empty($grade_arr['D'])) $grade_arr['D'] = 0;
        if(empty($grade_arr['E'])) $grade_arr['E'] = 0;
        if(empty($grade_arr['F'])) $grade_arr['F'] = 0;

    }

    $tNonIndex = $tCountNon200_ni + $robot_disallow;

    arsort($page_errors);

    $tot_errors = 0;
    // foreach ($final_arr as $pkey => $pvalue) {
    //     if(!empty($pvalue) && !empty($page_errors[$pkey]) && $page_errors[$pkey] > 0)
    //     {
    //         $tot_errors += 1;
    //         if(strstr(T_ERR_MAJOR, $pkey))
    //             $err_major[$pkey] = $pvalue;
    //         if(strstr(T_ERR_MEDIUM, $pkey))
    //             $err_medium[$pkey] = $pvalue;
    //         if(strstr(T_ERR_MINOR, $pkey))
    //             $err_minor[$pkey] = $pvalue;
    //     }
    // }

    foreach ($page_errors as $pkey => $pvalue) {
        $history_errors[$pkey] = 0;
        if(strstr(T_ERR_MAJOR, $pkey)) {
            $err_major[$pkey] = $final_arr[$pkey];
            $tot_errors += 1;
        }
    }
    foreach ($page_errors as $pkey => $pvalue) {
        if(strstr(T_ERR_MEDIUM, $pkey)) {
            $err_medium[$pkey] = $final_arr[$pkey];
            $tot_errors += 1;
        }
    }
    foreach ($page_errors as $pkey => $pvalue) {
        if(strstr(T_ERR_MINOR, $pkey)) {
            $err_minor[$pkey] = $final_arr[$pkey];
            $tot_errors += 1;
        }
    }

    $his_data = getHistoryData(P_ID, $my_dist_host, $my_dist_user, $my_dist_pass, $my_dist_db);
    foreach ($history_errors as $hiskey => $hisvalue) {
        if(isset($his_data[$hiskey])) {
            if(empty($his_data[$hiskey])) {
                $history_value = 0;
            } else {
                $history_value = $his_data[$hiskey];
            }
            $history_errors[$hiskey] = $history_value;
        } else {
            $history_errors[$hiskey] = "new";
        }
    }
    $history_errors["total_crawled"] = $his_data['t_crawled'];
    $history_errors["total_non200"] = $his_data['t_non200'];
    $history_errors["total_nonindex"] = $his_data['t_nonindex'];
    $history_errors["total_failed"] = $his_data['t_failed'];

    $page_info = array(
        'total_scraped' => $totalScraped,
        'non200' => $tCountNon200,
        'nonindex' => $tNonIndex,
        'failed_urls' => $tCountFailed,
        'total_errors' => $tot_errors,
        'last_checked_on' => $last_checked_on,
        'total_quota_used' => $total_quota_used
    );

    // $u_agents = array(
    //     'Mozilla/5.0 (compatible; 008/0.83; http://www.80legs.com/webcrawler.html) Gecko/2008032620',
    //     'Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)',
    //     'Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)',
    //     'Mozilla/4.0 (compatible: FDSE robot)',
    //     'Gigabot/3.0 (http://www.gigablast.com/spider.html)',
    //     'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)',
    //     'Googlebot/2.1 (+http://www.googlebot.com/bot.html)',
    //     'Mozilla/5.0 (compatible; MJ12bot/v1.2.4; http://www.majestic12.co.uk/bot.php?+)',
    //     'Mozilla/5.0 (compatible; MojeekBot/2.0; http://www.mojeek.com/bot.html)',
    //     'msnbot/2.1'
    // );

    $page = array();
    $page['page_info'] = json_encode($page_info);
    $page['err_major'] = json_encode($err_major);
    $page['err_medium'] = json_encode($err_medium);
    $page['err_minor'] = json_encode($err_minor);
    $page['grade_arr'] = json_encode($grade_arr);
    $page['page_errors'] = json_encode($page_errors);
    $page['url_error'] = json_encode($url_error);
    $page['url_grade'] = json_encode($url_grade);
    $page['url_pos'] = json_encode($url_pos);
    $page['history_errors'] = json_encode($history_errors);
    $page['err_major_count'] = count(explode(",", substr(trim(T_ERR_MAJOR), 1, -1)));
    $page['err_medium_count'] = count(explode(",", substr(trim(T_ERR_MEDIUM), 1, -1)));
    $page['err_minor_count'] = count(explode(",", substr(trim(T_ERR_MINOR), 1, -1)));

    $page['response_code'] = 100;

    echo json_encode($page);

}



function getAllProjPages($uid, $pid, $my_dist_host, $my_dist_user, $my_dist_pass, $my_dist_db)
{
    $dist_db = conDist($my_dist_host, $my_dist_user, $my_dist_pass, $my_dist_db);

    $qry = "SELECT SUM(pages_to_crawl) AS total_pages FROM projects WHERE user_id=$uid AND expired=0";

    if($chk = mysqli_query($dist_db, $qry)) {
        if(mysqli_num_rows($chk)) {
            $res = mysqli_fetch_assoc($chk);
            if(!empty($res['total_pages'])) {
                return $res['total_pages'];
            }
        }
    }
}

function getHistoryData($pid, $my_dist_host, $my_dist_user, $my_dist_pass, $my_dist_db)
{
    $res = array();
    $dist_db = conDist($my_dist_host, $my_dist_user, $my_dist_pass, $my_dist_db);

    $qry = "SELECT * FROM history WHERE p_id=$pid ORDER BY created_on DESC LIMIT 1,1";
    if($chk = mysqli_query($dist_db, $qry)) {
        if(mysqli_num_rows($chk)) {
            $res = mysqli_fetch_assoc($chk);
            unset($res['created_on']);
            unset($res['id']);
            unset($res['p_id']);
            unset($res['week']);
        }
    }
    return $res;
}