
<?php
    error_reporting(E_ALL);
    ini_set('display_errors', 'On');

    ini_set('memory_limit', '1024M');
    set_time_limit(5000);
    // if website form submitted for checking duplicate content
if (!empty($_GET['project'])) {
    require_once "conf.php"; //common configuration constants and error function defined

    $stime = time();

    $page_info = array();
    $err_major = array();
    $err_medium = array();
    $err_minor = array();
    $grade_arr = array();
    $page_errors = array();
    $url_error = array();
    $history_errors = array();
    $paged_data = array();

    $c_err_major = 0;
    $c_err_medium = 0;
    $c_err_minor = 0;
    $tCountNon200 = 0;
    $tCountNon200_ni = 0;
    $cache_error = 0;
    $total_paged_pages = 0;

    $page_limit = isset($_GET['page_limit'])?$_GET['page_limit']:10;
    $page_offset = isset($_GET['page_offset'])?$_GET['page_offset']:0;
    $page_search = isset($_GET['page_search'])?$_GET['page_search']:"";
    $page_sort_field = isset($_GET['page_sort_field'])?$_GET['page_sort_field']:"";
    $page_sort_value = isset($_GET['page_sort_value'])?$_GET['page_sort_value']:"";

    $proj_data = getProjData($my_dist_host, $my_dist_user, $my_dist_pass, $my_dist_db);
    $proj_data_orig = getOrigProjData($my_dist_host, $my_dist_user, $my_dist_pass, $my_dist_db);
    $week = (int)$proj_data['crawl_no'] - 1;
    $last_checked_on = $proj_data['last_crawled_on'];
    $user_id = $proj_data['user_id'];

    $cache_dir = "./Caches";
    $cache_pid_dir = "./Caches/".P_ID;
    $cache_file = $cache_pid_dir."/".$proj_data_orig['id']."_index_".$proj_data_orig['crawl_no'];

    if(!is_dir($cache_dir)) {
        if (!mkdir($cache_dir)  || !chmod($cache_dir, 0777)) {
            $cache_error = 1;
        }
    }
    if(!is_dir($cache_pid_dir)) {
        if (!mkdir($cache_pid_dir)  || !chmod($cache_pid_dir, 0777)) {
            $cache_error = 1;
        }
    }

$cache_error = 1;
    if(file_exists($cache_file) && empty($cache_error)) {
        $final_data = array();

        $cache_open = fopen($cache_file, "r");
        while($cache_data = fgets($cache_open)) {
            $cd_str = array();
            $f_cache_data = "";

            if(!empty($cache_data)) {
                $cd_str = explode("-->", $cache_data);
                if($cd_str[0] == "url_error") {
                    $data_decoded = json_decode($cd_str[1], true);
                    foreach ($data_decoded as $dkey => $dvalue) {
                        if(!empty($page_search)) {
                            $dkey_decoded = urldecode(urldecode(urldecode($dkey)));
                            if(stristr($dkey_decoded, $page_search) || stristr($data_decoded[$dkey]['title'], $page_search)) {
                                $paged_data[$dkey] = $dvalue;
                                if(!empty($page_sort_field)) {
                                    if($page_sort_field == "title") {
                                        $sort_fields[] = $data_decoded[$dkey]['title'];
                                    } elseif($page_sort_field == "errors") {
                                        $sort_fields[] = $data_decoded[$dkey]['errors'];
                                    } elseif($page_sort_field == "grade") {
                                        $sort_fields[] = $data_decoded[$dkey]['grade'];
                                    }
                                }
                            }
                        } else {
                            $paged_data[$dkey] = $dvalue;
                            if(!empty($page_sort_field)) {
                                if($page_sort_field == "title") {
                                    $sort_fields[] = $data_decoded[$dkey]['title'];
                                } elseif($page_sort_field == "errors") {
                                    $sort_fields[] = $data_decoded[$dkey]['errors'];
                                } elseif($page_sort_field == "grade") {
                                    $sort_fields[] = $data_decoded[$dkey]['grade'];
                                }
                            }
                        }
                    }

                    if(!empty($page_sort_field)) {
                        if($page_sort_value == "desc") {
                            array_multisort($sort_fields, SORT_DESC, $paged_data);
                        } elseif($page_sort_value == "asc") {
                            array_multisort($sort_fields, SORT_ASC, $paged_data);
                        }
                    }
                    $total_paged_pages = count($paged_data);

                    $f_cache_data = json_encode(array_slice($paged_data, $page_offset, $page_limit));
                } else {
                    $f_cache_data = $cd_str[1];
                }
                $final_data[$cd_str[0]] = $f_cache_data;
            }
        }

        if(isset($final_data['page_info'])) {
            $info_decoded = json_decode($final_data['page_info'], true);
            $info_decoded['total_pages'] = $total_paged_pages;
            $final_data['page_info'] = json_encode($info_decoded);
        }
        echo json_encode($final_data);
        exit;
    } else {
        clearDirs($cache_pid_dir."/");
    }






    $total_quota_used = getAllProjPages($proj_data_orig['user_id'], $proj_data_orig['id'], $my_dist_host, $my_dist_user, $my_dist_pass, $my_dist_db);

    $totalScraped = $mongoDB->auditor->count(
        array('pi' => P_ID, 'w' => $week)
    );

    $tCountNon200 = $mongoDB->auditor_non200->count(
        array('pi' => P_ID, 'w' => $week)
    );

    $tCountNon200_ni = $mongoDB->auditor_nonindex->count(
        array('pi' => P_ID, 'w' => $week)
    );

    $tCountFailed = $mongoDB->auditor_failed->count(
        array('pi' => P_ID, 'w' => $week)
    );

    if($proj_data['runnow_priority'] == 99 || ($proj_data['pages_to_crawl'] > $proj_data_orig['pages_to_crawl'])) {
        $red_data = getReducesData($totalScraped, $tCountNon200, $tCountNon200_ni, $tCountFailed, $proj_data_orig['pages_to_crawl']);

        if(!empty($red_data['unique'])) {
            $totalScraped = $red_data['unique'];
        }
        if(!empty($red_data['non200'])) {
            $tCountNon200 = $red_data['non200'];
        }
        if(!empty($red_data['nonindex'])) {
            $tCountNon200_ni = $red_data['nonindex'];
        }
        if(!empty($red_data['failed'])) {
            $tCountFailed = $red_data['failed'];
        }
    }


    $his_data = getHistoryData(P_ID, $week, $err_col_arr, $proj_data, $proj_data_orig, $my_dist_host, $my_dist_user, $my_dist_pass, $my_dist_db, $mongoDB);


    $page_errors = array();
    $e_err_arr = array();
    $robot_disallow = 0;

    // $robot_content = file_get_contents('http://'.URL_TO_CRAWL."/robots.txt");
    // preg_match_all('/disallow(:?)(.*?)\n/i', $robot_content, $m);

    $err_col_arr['_id'] = 0; $err_col_arr['u'] = $err_col_arr['pos'] = $err_col_arr['t'] = $err_col_arr['etit.tit'] = 1;

    $each_errors = $mongoDB->auditor->find(
        array('pi' => P_ID, 'w' => $week),
        $err_col_arr
    );
    $each_errors->limit((int)$totalScraped);

    $url_arr = array();

    while($e_error = $each_errors->getNext())
    {
        $e_err_arr[] = $e_error;
    }

    $totalScraped = $totalScraped + $tCountNon200 + $tCountNon200_ni + $tCountFailed;

    if(empty($totalScraped))
    {
        $result = json_encode(
            array(
                'response_code' => 101,
                'response_message' => "No Data Found for Project ". URL_TO_CRAWL
            )
        );

        $dash_cache = fopen($cache_file, "w");
        fwrite($dash_cache, "response_code-->101"."\n");
        fwrite($dash_cache, "response_message-->No Data Found for Project ". URL_TO_CRAWL."\n");
        fclose($dash_cache);

        echo $result; exit;
    }


    foreach ($e_err_arr as $erval) {
        $each_url_arr = array();
        foreach ($erval as $eskey => $esvalue) {

            if($eskey !== "u" && $eskey !== "pos" && $eskey !== 't')
            {
                if(is_array($esvalue))
                {
                    foreach ($esvalue as $esk => $esv) {
                        if(!empty($esvalue[$esk]) && $esvalue[$esk] >= 1 && $esk !== 'tit')
                        {
                            if(empty($page_errors[$esk]))
                                $page_errors[$esk] = 0;
                            $page_errors[$esk] += 1;
                            $each_url_arr[$esk] = 1;
                        }
                    }
                }
                elseif(!empty($erval[$eskey]) && $erval[$eskey] >= 1 && $eskey !== 'tit')
                {
                    if(empty($page_errors[$eskey]))
                        $page_errors[$eskey] = 0;
                    $page_errors[$eskey] += 1;
                    $each_url_arr[$eskey] = 1;
                }
            }

        }

        if($erval['t'] > T_SPEED_SLOW)
        {
            if(empty($page_errors['t']))
                $page_errors['t'] = 0;

            $page_errors['t'] += 1;
        }

        $erval['etit']['tit'] = empty($erval['etit']['tit']) ? "" : $erval['etit']['tit'];

        $erval['u'] = rawurlencode($erval['u']);
        $e_grade = calculateGrade($each_url_arr);
        $url_grade[] = $e_grade;
        $c_e_url_arr = count($each_url_arr);
        $sort_fields[] = $e_grade;

        $url_error[$erval['u']]['title'] = $erval['etit']['tit'];
        $url_error[$erval['u']]['errors'] = $c_e_url_arr;
        $url_error[$erval['u']]['grade'] = $e_grade;
        $url_error[$erval['u']]['pos'] = $erval['pos'];
    }

    array_multisort($sort_fields, SORT_ASC, $url_error);

    $grade_arr = array_count_values($url_grade);

    if(empty($grade_arr['A'])) $grade_arr['A'] = 0;
    if(empty($grade_arr['B'])) $grade_arr['B'] = 0;
    if(empty($grade_arr['C'])) $grade_arr['C'] = 0;
    if(empty($grade_arr['D'])) $grade_arr['D'] = 0;
    if(empty($grade_arr['E'])) $grade_arr['E'] = 0;
    if(empty($grade_arr['F'])) $grade_arr['F'] = 0;

    $tNonIndex = $tCountNon200_ni + $robot_disallow;

    arsort($page_errors);

    $tot_errors = 0;
    foreach ($page_errors as $pkey => $pvalue) {
        $history_errors[$pkey] = 0;
        if(strstr(T_ERR_MAJOR, $pkey)) {
            $err_major[$pkey] = $final_arr[$pkey];
            $tot_errors += 1;
        }
        elseif(strstr(T_ERR_MEDIUM, $pkey)) {
            $err_medium[$pkey] = $final_arr[$pkey];
            $tot_errors += 1;
        }
        elseif(strstr(T_ERR_MINOR, $pkey)) {
            $err_minor[$pkey] = $final_arr[$pkey];
            $tot_errors += 1;
        }
    }

    foreach ($history_errors as $hiskey => $hisvalue) {
        if(isset($his_data[$hiskey])) {
            if(empty($his_data[$hiskey])) {
                $history_value = 0;
            } else {
                $history_value = $his_data[$hiskey];
            }
            $history_errors[$hiskey] = $history_value;
        } else {
            $history_errors[$hiskey] = "new";
        }
    }
    $history_errors["total_crawled"] = isset($his_data['t_crawled'])?$his_data['t_crawled']:0;
    $history_errors["total_non200"] = isset($his_data['t_non200'])?$his_data['t_non200']:0;
    $history_errors["total_nonindex"] = isset($his_data['t_nonindex'])?$his_data['t_nonindex']:0;
    $history_errors["total_failed"] = isset($his_data['t_failed'])?$his_data['t_failed']:0;

    $page_info = array(
        'total_scraped' => $totalScraped,
        'non200' => $tCountNon200,
        'nonindex' => $tNonIndex,
        'failed_urls' => $tCountFailed,
        'total_errors' => $tot_errors,
        'last_checked_on' => $last_checked_on,
        'total_quota_used' => $total_quota_used,
        'total_pages' => count($url_error)
    );


    $c_err_major = count(explode(",", substr(trim(T_ERR_MAJOR), 1, -1)));
    $c_err_medium = count(explode(",", substr(trim(T_ERR_MEDIUM), 1, -1)));
    $c_err_minor = count(explode(",", substr(trim(T_ERR_MINOR), 1, -1)));

    $dash_cache = fopen($cache_file, "w");
    fwrite($dash_cache, "page_info-->".json_encode($page_info)."\n");
    fwrite($dash_cache, "err_major-->".json_encode($err_major)."\n");
    fwrite($dash_cache, "err_medium-->".json_encode($err_medium)."\n");
    fwrite($dash_cache, "err_minor-->".json_encode($err_minor)."\n");
    fwrite($dash_cache, "grade_arr-->".json_encode($grade_arr)."\n");
    fwrite($dash_cache, "page_errors-->".json_encode($page_errors)."\n");
    fwrite($dash_cache, "url_error-->".json_encode($url_error)."\n");
    fwrite($dash_cache, "history_errors-->".json_encode($history_errors)."\n");
    fwrite($dash_cache, "err_major_count-->".$c_err_major."\n");
    fwrite($dash_cache, "err_medium_count-->".$c_err_medium."\n");
    fwrite($dash_cache, "err_minor_count-->".$c_err_minor."\n");
    fwrite($dash_cache, "response_code-->100"."\n");
    fclose($dash_cache);

    $page = array();
    $page['page_info'] = json_encode($page_info);
    $page['err_major'] = json_encode($err_major);
    $page['err_medium'] = json_encode($err_medium);
    $page['err_minor'] = json_encode($err_minor);
    $page['grade_arr'] = json_encode($grade_arr);
    $page['page_errors'] = json_encode($page_errors);
    $page['url_error'] = json_encode(array_slice($url_error, $page_offset, $page_limit));
    $page['history_errors'] = json_encode($history_errors);
    $page['err_major_count'] = $c_err_major;
    $page['err_medium_count'] = $c_err_medium;
    $page['err_minor_count'] = $c_err_minor;

    $page['response_code'] = 100;

    echo json_encode($page);

}



function getAllProjPages($uid, $pid, $my_dist_host, $my_dist_user, $my_dist_pass, $my_dist_db)
{
    $dist_db = conDist($my_dist_host, $my_dist_user, $my_dist_pass, $my_dist_db);

    $qry = "SELECT SUM(pages_to_crawl) AS total_pages FROM projects WHERE user_id=$uid AND expired=0";

    if($chk = mysqli_query($dist_db, $qry)) {
        if(mysqli_num_rows($chk)) {
            $res = mysqli_fetch_assoc($chk);
            if(!empty($res['total_pages'])) {
                return $res['total_pages'];
            }
        }
    }
}

function getHistoryData($pid, $week, $err_arr, $proj_data, $proj_data_orig, $my_dist_host, $my_dist_user, $my_dist_pass, $my_dist_db, $mongoDB)
{
    $week = $week - 1;
    // $res = array();
    // $dist_db = conDist($my_dist_host, $my_dist_user, $my_dist_pass, $my_dist_db);

    // $qry = "SELECT * FROM history WHERE p_id=$pid ORDER BY created_on DESC LIMIT 1,1";
    // if($chk = mysqli_query($dist_db, $qry)) {
    //     if(mysqli_num_rows($chk)) {
    //         $res = mysqli_fetch_assoc($chk);
    //         unset($res['created_on']);
    //         unset($res['id']);
    //         unset($res['p_id']);
    //         unset($res['week']);
    //     }
    // }
    // return $res;

    foreach ($err_arr as $erkey => $ervalue) {
        $page_errors[$erkey] = 0;
    }

    $totalScraped = $mongoDB->auditor->count(
        array('pi' => P_ID, 'w' => $week)
    );

    $tCountNon200 = $mongoDB->auditor_non200->count(
        array('pi' => P_ID, 'w' => $week)
    );

    $tCountNon200_ni = $mongoDB->auditor_nonindex->count(
        array('pi' => P_ID, 'w' => $week)
    );

    $tCountFailed = $mongoDB->auditor_failed->count(
        array('pi' => P_ID, 'w' => $week)
    );

    if($proj_data['runnow_priority'] == 99 || ($proj_data['pages_to_crawl'] > $proj_data_orig['pages_to_crawl'])) {
        $red_data = getReducesData($totalScraped, $tCountNon200, $tCountNon200_ni, $tCountFailed, $proj_data_orig['pages_to_crawl']);

        if(!empty($red_data['unique'])) {
            $totalScraped = $red_data['unique'];
        }
        if(!empty($red_data['non200'])) {
            $tCountNon200 = $red_data['non200'];
        }
        if(!empty($red_data['nonindex'])) {
            $tCountNon200_ni = $red_data['nonindex'];
        }
        if(!empty($red_data['failed'])) {
            $tCountFailed = $red_data['failed'];
        }
    }
    $page_errors['t_non200'] = $tCountNon200;
    $page_errors['t_nonindex'] = $tCountNon200_ni;
    $page_errors['t_failed'] = $tCountFailed;
    $page_errors['t_crawled'] = $totalScraped + $tCountNon200 + $tCountNon200_ni + $tCountFailed;

    $err_col_arr['_id'] = 0;

    $each_errors = $mongoDB->auditor->find(
        array('pi' => P_ID, 'w' => $week),
        $err_col_arr
    );
    $each_errors->limit((int)$totalScraped);

    while($e_error = $each_errors->getNext())
    {
        $e_err_arr[] = $e_error;
    }

    foreach ($e_err_arr as $erval) {
        $each_url_arr = array();
        foreach ($erval as $eskey => $esvalue) {
            if(is_array($esvalue)) {
                foreach ($esvalue as $esk => $esv) {
                    if(!empty($esvalue[$esk]) && $esvalue[$esk] >= 1) {
                        if(empty($page_errors[$esk]))
                            $page_errors[$esk] = 0;
                        $page_errors[$esk] += 1;
                    }
                }
            }
            elseif(!empty($erval[$eskey]) && $erval[$eskey] >= 1) {
                if(empty($page_errors[$eskey]))
                    $page_errors[$eskey] = 0;
                $page_errors[$eskey] += 1;
            }
        }

        if($erval['t'] > T_SPEED_SLOW)
        {
            if(empty($page_errors['t']))
                $page_errors['t'] = 0;

            $page_errors['t'] += 1;
        }
    }
    return $page_errors;
}

function clearDirs($dir)
{
    if (!empty($dir) && is_dir($dir)) {
        $oldDups = array_diff(scandir($dir), array('.', '..'));
        foreach ($oldDups as $oldFile) {
            $oldFilePath = $dir.$oldFile;
            if (file_exists("$oldFilePath")) {
                unlink("$oldFilePath");
            }
        }
    }
}