
<?php
    //error_reporting(E_ALL);
    //ini_set('display_errors', 'On');
    ini_set('memory_limit', '2048M');
    set_time_limit(5000);
    // if website form submitted for checking duplicate content
if (!empty($_GET['project'])) {
    require_once "conf.php"; //common configuration constants and error function defined

    $stime = time();

    $page_info = array();
    $err_major = array();
    $err_medium = array();
    $err_minor = array();
    $grade_arr = array();
    $page_errors = array();
    $url_error = array();
    $history_errors = array();
    $paged_data = array();

    $c_err_major = 0;
    $c_err_medium = 0;
    $c_err_minor = 0;
    $tCountNon200 = 0;
    $tCountNon200_ni = 0;
    $cache_error = 0;
    $total_paged_pages = 0;

    $page_limit = isset($_GET['page_limit'])?$_GET['page_limit']:10;
    $page_offset = isset($_GET['page_offset'])?$_GET['page_offset']:0;
    $page_search = isset($_GET['page_search'])?$_GET['page_search']:"";
    $page_sort_field = isset($_GET['page_sort_field'])?$_GET['page_sort_field']:"grade";
    $page_sort_value = isset($_GET['page_sort_value'])?$_GET['page_sort_value']:"asc";

    $proj_data = getProjData($my_dist_host, $my_dist_user, $my_dist_pass, $my_dist_db);

    $proj_data_orig = getOrigProjData($my_dist_host, $my_dist_user, $my_dist_pass, $my_dist_db);

    #######################################################
    $ca_id = (int)$proj_data['pauditor_id'] ;
    $pa_id = (int)$proj_data['p2auditor_id'] ;
    #######################################################


    $user_id = $proj_data['user_id'];
    $cache_dir = "./Caches";
    $cache_pid_dir = "./Caches/".P_ID;
    $cache_file = $cache_pid_dir."/".$proj_data_orig['p_id']."_index_".$proj_data_orig['crawl_no'];

    if(!is_dir($cache_dir)) {
        if (!mkdir($cache_dir)  || !chmod($cache_dir, 0777)) {
            $cache_error = 1;
        }
    }
    if(!is_dir($cache_pid_dir)) {
        if (!mkdir($cache_pid_dir)  || !chmod($cache_pid_dir, 0777)) {
            $cache_error = 1;
        }
    }
    $cache_error = 1;

    /*
    if(file_exists($cache_file) && empty($cache_error)) {
        $final_data = array();

        $cache_open = fopen($cache_file, "r");
        while($cache_data = fgets($cache_open)) {
            $cd_str = array();
            $f_cache_data = "";
            if(!empty($cache_data)) {
                $cd_str = explode("-->", $cache_data);
                if($cd_str[0] == "url_error") {
                    $data_decoded = json_decode($cd_str[1], true);
                    foreach ($data_decoded as $dkey => $dvalue) {
                        if(!empty($page_search)) {
                            $dkey_decoded = urldecode(urldecode(urldecode($dkey)));
                            if(stristr($dkey_decoded, $page_search) || stristr($data_decoded[$dkey]['title'], $page_search)) {
                                $paged_data[$dkey] = $dvalue;
                                if(!empty($page_sort_field)) {
                                    if($page_sort_field == "title") {
                                        $sort_fields[] = $data_decoded[$dkey]['title'];
                                    } elseif($page_sort_field == "errors") {
                                        $sort_fields[] = $data_decoded[$dkey]['errors'];
                                    } elseif($page_sort_field == "grade") {
                                        $sort_fields[] = $data_decoded[$dkey]['grade'];
                                    }
                                }
                            }
                        } else {
                            $paged_data[$dkey] = $dvalue;
                            if(!empty($page_sort_field)) {
                                if($page_sort_field == "title") {
                                    $sort_fields[] = $data_decoded[$dkey]['title'];
                                } elseif($page_sort_field == "errors") {
                                    $sort_fields[] = $data_decoded[$dkey]['errors'];
                                } elseif($page_sort_field == "grade") {
                                    $sort_fields[] = $data_decoded[$dkey]['grade'];
                                }
                            }
                        }
                    }

                    if(!empty($page_sort_field)) {
                        if($page_sort_value == "desc") {
                            array_multisort($sort_fields, SORT_DESC, $paged_data);
                        } elseif($page_sort_value == "asc") {
                            array_multisort($sort_fields, SORT_ASC, $paged_data);
                        }
                    }
                    $total_paged_pages = count($paged_data);

                    $f_cache_data = json_encode(array_slice($paged_data, $page_offset, $page_limit));
                } else {
                    $f_cache_data = $cd_str[1];
                }
                $final_data[$cd_str[0]] = $f_cache_data;
            }
        }

        if(isset($final_data['page_info'])) {
            $info_decoded = json_decode($final_data['page_info'], true);
            $info_decoded['total_pages'] = $total_paged_pages;
            $final_data['page_info'] = json_encode($info_decoded);
        }
        echo json_encode($final_data);
        exit;
    } else {
        clearDirs($cache_pid_dir."/");
    }*/

    //TODO uncomment after quota implementation Commented for now
    //$total_quota_used = getAllProjPages($proj_data_orig['user_id'], $proj_data_orig['p_id'], $my_dist_host, $my_dist_user, $my_dist_pass, $my_dist_db);
    # Get total scraped in current week
    $totalScraped = $mongoDB->auditor->count(
        array('p_id' => P_ID, 'a_id'=> $ca_id)
    );
    $tCountNon200 = $mongoDB->auditor_status->count(
        array('p_id' => P_ID, 'a_id'=>$ca_id ,'$or' => 
                    array(
                        array("status" => array('$lt'=>200)),
                        array("status" => array('$gt'=>400,'$lt'=>1000))
                    )
                )
    );

    $tCountNon200_ni = $mongoDB->auditor_status->count(
        array('p_id' => P_ID, 'a_id'=>$ca_id,'status'=>1000)
    );


    $tCountFailed = $mongoDB->auditor_status->count(
        array('p_id' => P_ID, 'a_id'=>$ca_id,'status'=>1001)
    );

    $tCountRedirected = $mongoDB->au_redirected->count(
        array('p_id'=> P_ID ,'a_id'=>$ca_id)
    );

    /*
    if($proj_data['pages_to_crawl'] > $proj_data_orig['pages_to_crawl']) {
        $red_data = getReducesData($totalScraped, $tCountNon200, $tCountNon200_ni, $tCountFailed, $proj_data_orig['pages_to_crawl']);

        if(!empty($red_data['unique'])) {
            $totalScraped = $red_data['unique'];
        }
        if(!empty($red_data['non200'])) {
            $tCountNon200 = $red_data['non200'];
        }
        if(!empty($red_data['nonindex'])) {
            $tCountNon200_ni = $red_data['nonindex'];
        }
        if(!empty($red_data['failed'])) {
            $tCountFailed = $red_data['failed'];
        }
    }
    */

    $his_data = getHistoryData(P_ID, $pa_id, $err_col_arr, $proj_data, $proj_data_orig, $my_dist_host, $my_dist_user, $my_dist_pass, $my_dist_db, $mongoDB);


    $page_errors = array();
    $e_err_arr = array();
    $robot_disallow = 0;

    $err_col_arr['_id'] = 0; $err_col_arr['url'] = $err_col_arr['pos'] = $err_col_arr['t'] = $err_col_arr['etit.tit'] = 1;

    $each_errors = $mongoDB->auditor->find(
        array('p_id' => P_ID, 'a_id'=>$ca_id),
        $err_col_arr
    );
    //$each_errors->limit((int)$totalScraped);

    $url_arr = array();

    while($e_error = $each_errors->getNext())
    {
        $e_err_arr[] = $e_error;
    }
    
    $totalCrawled = $totalScraped + $tCountNon200 + $tCountNon200_ni + $tCountFailed ;

    $eorph = getorphanUrls(P_ID,$ca_id,$mongoDB);

    $can_count = $mongoDB->au_canonical->count(array('p_id'=> P_ID , 'a_id'=> $ca_id));

    if(empty($totalCrawled))
    {
        $result = json_encode(
            array(
                'response_code' => 101,
                'response_message' => "No Data Found for Project ". URL_TO_CRAWL
            )
        );

        /*$dash_cache = fopen($cache_file, "w");
        fwrite($dash_cache, "response_code-->101"."\n");
        fwrite($dash_cache, "response_message-->No Data Found for Project ". URL_TO_CRAWL."\n");
        fclose($dash_cache);*/

        echo $result; exit;
    }
    foreach ($e_err_arr as $erval) {
        $each_url_arr = array();
        foreach ($erval as $eskey => $esvalue) {
            if($eskey !== "url" && $eskey !== "pos" && $eskey !== 't') {
                if(is_array($esvalue)) {
                    foreach ($esvalue as $esk => $esv) {
                        if(!empty($esvalue[$esk]) && $esvalue[$esk] >= 1 && $esk !== 'tit') {
                            if(empty($page_errors[$esk])) {
                                $page_errors[$esk] = 0;
                            }
                            $page_errors[$esk] += 1;
                            $each_url_arr[$esk] = 1;
                        }
                    }
                }
                elseif(!empty($erval[$eskey]) && $erval[$eskey] >= 1 && $eskey !== 'tit' && $eskey !== 'scheme') {
                    if(empty($page_errors[$eskey])) {
                        $page_errors[$eskey] = 0;
                    }
                    $page_errors[$eskey] += 1;
                    $each_url_arr[$eskey] = 1;
                }
            }

        }
        $ehrfl = getHrefData(P_ID,$ca_id,$erval['url'],$mongoDB);
        if ($ehrfl)
        {
            $page_errors['ehrfl'] += 1;
            $each_url_arr['ehrfl'] = 1;
        }

        if($erval['t'] > T_SPEED_SLOW) {
            if(empty($page_errors['t'])) {
                    $page_errors['t'] = 0;
            }
            $page_errors['t'] += 1;
        }

        $erval['etit']['tit'] = empty($erval['etit']['tit']) ? "" : $erval['etit']['tit'];
        $erval['url'] = rawurlencode($erval['url']);
        $e_grade = calculateGrade($each_url_arr);
        $url_grade[] = $e_grade;
        $c_e_url_arr = count($each_url_arr);

        $url_error[$erval['url']]['title'] = $erval['etit']['tit'];
        $url_error[$erval['url']]['errors'] = $c_e_url_arr;
        $url_error[$erval['url']]['grade'] = $e_grade;
        $url_error[$erval['url']]['pos'] = $erval['pos'];
    }
    $grade_arr = array_count_values($url_grade);

    if(empty($grade_arr['A'])) $grade_arr['A'] = 0;
    if(empty($grade_arr['B'])) $grade_arr['B'] = 0;
    if(empty($grade_arr['C'])) $grade_arr['C'] = 0;
    if(empty($grade_arr['D'])) $grade_arr['D'] = 0;
    if(empty($grade_arr['E'])) $grade_arr['E'] = 0;
    if(empty($grade_arr['F'])) $grade_arr['F'] = 0;

    $tNonIndex = $tCountNon200_ni + $robot_disallow;

    arsort($page_errors);

    $tot_errors = 0;
    foreach ($page_errors as $pkey => $pvalue) {
        $history_errors[$pkey] = 0;
        if(strstr(T_ERR_MAJOR, $pkey)) {
            $err_major[$pkey] = $final_arr[$pkey];
            $tot_errors += 1;
        }
        elseif(strstr(T_ERR_MEDIUM, $pkey)) {
            $err_medium[$pkey] = $final_arr[$pkey];
            $tot_errors += 1;
        }
        elseif(strstr(T_ERR_MINOR, $pkey)) {
            $err_minor[$pkey] = $final_arr[$pkey];
            $tot_errors += 1;
        }
    }

    foreach ($history_errors as $hiskey => $hisvalue) {
        if(isset($his_data[$hiskey])) {
            if(empty($his_data[$hiskey])) {
                $history_value = 0;
            } else {
                $history_value = $his_data[$hiskey];
            }
            $history_errors[$hiskey] = $history_value;
        } else {
            $history_errors[$hiskey] = "new";
        }
    }
    $history_errors["total_crawled"] = isset($his_data['t_crawled'])?$his_data['t_crawled']:0;
    $history_errors["total_non200"] = isset($his_data['t_non200'])?$his_data['t_non200']:0;
    $history_errors["total_nonindex"] = isset($his_data['t_nonindex'])?$his_data['t_nonindex']:0;
    $history_errors["total_failed"] = isset($his_data['t_failed'])?$his_data['t_failed']:0;

    if($proj_data['dup_running'] == 97 && $proj_data['runnow_priority'] == 90) {
        $proj_data['runnow_priority'] = 97;
    }

    $page_info = array(
        'total_scraped' => $totalCrawled,
        'message_status' => $proj_data['runnow_priority'],
        'non200' => $tCountNon200,
        'nonindex' => $tNonIndex,
        'failed_urls' => $tCountFailed,
        'total_errors' => $tot_errors,
        'last_checked_on' => date("Y-m-d", $proj_data['pauditor_id']),
        'total_quota_used' => $total_quota_used,
        'total_pages' => count($url_error),
        'redirected_urls' => $tCountRedirected,
        'orphaned_sitemap_urls' => (!$eorph) ? 0 : count($eorph),
        'canonical_urls' => $can_count
    );


    $c_err_major = count(explode(",", substr(trim(T_ERR_MAJOR), 1, -1)));
    $c_err_medium = count(explode(",", substr(trim(T_ERR_MEDIUM), 1, -1)));
    $c_err_minor = count(explode(",", substr(trim(T_ERR_MINOR), 1, -1)));

    // $dash_cache = fopen($cache_file, "w");
    // fwrite($dash_cache, "page_info-->".json_encode($page_info)."\n");
    // fwrite($dash_cache, "err_major-->".json_encode($err_major)."\n");
    // fwrite($dash_cache, "err_medium-->".json_encode($err_medium)."\n");
    // fwrite($dash_cache, "err_minor-->".json_encode($err_minor)."\n");
    // fwrite($dash_cache, "grade_arr-->".json_encode($grade_arr)."\n");
    // fwrite($dash_cache, "page_errors-->".json_encode($page_errors)."\n");
    // fwrite($dash_cache, "url_error-->".json_encode($url_error)."\n");
    // fwrite($dash_cache, "history_errors-->".json_encode($history_errors)."\n");
    // fwrite($dash_cache, "err_major_count-->".$c_err_major."\n");
    // fwrite($dash_cache, "err_medium_count-->".$c_err_medium."\n");
    // fwrite($dash_cache, "err_minor_count-->".$c_err_minor."\n");
    // fwrite($dash_cache, "response_code-->100"."\n");
    // fclose($dash_cache);


    foreach ($url_error as $dkey => $dvalue) {
        if(!empty($page_search)) {
            $dkey_decoded = urldecode(urldecode(urldecode($dkey)));
            if(stristr($dkey_decoded, $page_search) || stristr($url_error[$dkey]['title'], $page_search)) {
                $paged_data[$dkey] = $dvalue;
                if(!empty($page_sort_field)) {
                    if($page_sort_field == "title") {
                        $sort_fields[] = $url_error[$dkey]['title'];
                    } elseif($page_sort_field == "errors") {
                        $sort_fields[] = $url_error[$dkey]['errors'];
                    } elseif($page_sort_field == "grade") {
                        $sort_fields[] = $url_error[$dkey]['grade'];
                    }
                }
            }
        } else {
            $paged_data[$dkey] = $dvalue;
            if(!empty($page_sort_field)) {
                if($page_sort_field == "title") {
                    $sort_fields[] = $url_error[$dkey]['title'];
                } elseif($page_sort_field == "errors") {
                    $sort_fields[] = $url_error[$dkey]['errors'];
                } elseif($page_sort_field == "grade") {
                    $sort_fields[] = $url_error[$dkey]['grade'];
                }
            }
        }
    }
    $page_info['total_pages'] = count($paged_data);

    if(!empty($page_sort_field)) {
        if($page_sort_value == "desc") {
            array_multisort($sort_fields, SORT_DESC, $paged_data);
        } elseif($page_sort_value == "asc") {
            array_multisort($sort_fields, SORT_ASC, $paged_data);
        }
    }
    $url_error = array_slice($paged_data, $page_offset, $page_limit);
    $page = array();
    $page['page_info'] = json_encode($page_info);
    $page['err_major'] = json_encode($err_major);
    $page['err_medium'] = json_encode($err_medium);
    $page['err_minor'] = json_encode($err_minor);
    $page['all_page_data'] = json_encode($all_page_data);
    $page['grade_arr'] = json_encode($grade_arr);
    $page['page_errors'] = json_encode($page_errors);
    $page['url_error'] = json_encode($url_error);
    $page['history_errors'] = json_encode($history_errors); // Commented for now uncomment when difference is needed.
    $page['err_major_count'] = $c_err_major;
    $page['err_medium_count'] = $c_err_medium;
    $page['err_minor_count'] = $c_err_minor;

    $page['response_code'] = 100;

    echo json_encode($page);

}



function getAllProjPages($uid, $pid, $my_dist_host, $my_dist_user, $my_dist_pass, $my_dist_db)
{
    $dist_db = conDist($my_dist_host, $my_dist_user, $my_dist_pass, $my_dist_db);

    $qry = "SELECT SUM(pages_to_crawl) AS total_pages FROM sa_projects WHERE user_id=$uid AND expired=0";

    if($chk = mysqli_query($dist_db, $qry)) {
        if(mysqli_num_rows($chk)) {
            $res = mysqli_fetch_assoc($chk);
            if(!empty($res['total_pages'])) {
                return $res['total_pages'];
            }
        }
    }
}

function getHistoryData($pid, $week, $err_arr, $proj_data, $proj_data_orig, $my_dist_host, $my_dist_user, $my_dist_pass, $my_dist_db, $mongoDB)
{

    foreach ($err_arr as $erkey => $ervalue) {
        $page_errors[$erkey] = 0;
    }

    $totalScraped = $mongoDB->auditor->count(
        array('p_id' => P_ID,'a_id'=>$week)
    );
    
    $tCountNon200 = $mongoDB->auditor_status->count(
        array('p_id' => P_ID, 'a_id'=>$week,'$or' => 
                    array(
                        array("status" => array('$lt'=>200)),
                        array("status" => array('$gt'=>400,'$lt'=>1000))
                    )
                )
    );

    $tCountNon200_ni = $mongoDB->auditor_status->count(
        array('p_id' => P_ID,'a_id'=>$week,'status'=>1000)
    );

    $tCountFailed = $mongoDB->auditor_status->count(
        array('p_id' => P_ID, 'a_id'=>$week,'status'=>1001)
    );

    $tCountRedirected = $mongoDB->auditor_status->count(
        array('p_id'=> P_ID ,'a_id'=>(int)$week,"status" => array('$gt'=>300,'$lt'=>400))
    );

    /*if($proj_data['runnow_priority'] == 99 || ($proj_data['pages_to_crawl'] > $proj_data_orig['pages_to_crawl'])) {
        $red_data = getReducesData($totalScraped, $tCountNon200, $tCountNon200_ni, $tCountFailed, $proj_data_orig['pages_to_crawl']);

        if(!empty($red_data['unique'])) {
            $totalScraped = $red_data['unique'];
        }
        if(!empty($red_data['non200'])) {
            $tCountNon200 = $red_data['non200'];
        }
        if(!empty($red_data['nonindex'])) {
            $tCountNon200_ni = $red_data['nonindex'];
        }
        if(!empty($red_data['failed'])) {
            $tCountFailed = $red_data['failed'];
        }
    }*/
    $page_errors['t_non200'] = $tCountNon200;
    $page_errors['t_nonindex'] = $tCountNon200_ni;
    $page_errors['t_failed'] = $tCountFailed;
    $page_errors['t_Redirected'] =  $tCountRedirected;
    $page_errors['t_crawled'] = $totalScraped + $tCountNon200 + $tCountNon200_ni + $tCountFailed + $tCountRedirected;

    $err_col_arr['_id'] = 0;

    $each_errors = $mongoDB->auditor->find(
        array('p_id' => P_ID, 'a_id'=>$week),
        $err_col_arr
    );
    $each_errors->limit((int)$totalScraped);

    while($e_error = $each_errors->getNext())
    {
        $e_err_arr[] = $e_error;
    }

    if(!empty($e_err_arr)) {

        foreach ($e_err_arr as $erval) {
            $each_url_arr = array();
            foreach ($erval as $eskey => $esvalue) {
                if(is_array($esvalue)) {
                    foreach ($esvalue as $esk => $esv) {
                        if(!empty($esvalue[$esk]) && $esvalue[$esk] >= 1) {
                            if(empty($page_errors[$esk]))
                                $page_errors[$esk] = 0;
                            $page_errors[$esk] += 1;
                        }
                    }
                }
                elseif(!empty($erval[$eskey]) && $erval[$eskey] >= 1) {
                    if(empty($page_errors[$eskey]))
                        $page_errors[$eskey] = 0;
                    $page_errors[$eskey] += 1;
                }
            }

            if($erval['t'] > T_SPEED_SLOW)
            {
                if(empty($page_errors['t']))
                    $page_errors['t'] = 0;

                $page_errors['t'] += 1;
            }
        }
    }
    
    return $page_errors;
}

function clearDirs($dir)
{
    if (!empty($dir) && is_dir($dir)) {
        $oldDups = array_diff(scandir($dir), array('.', '..'));
        foreach ($oldDups as $oldFile) {
            $oldFilePath = $dir.$oldFile;
            if (file_exists("$oldFilePath")) {
                unlink("$oldFilePath");
            }
        }
    }
}

function getHrefData($p_id,$a_id,$url,$mongoDB){
    $ehrfl = 0;
    try{
        $href_curs = $mongoDB->hreflang_data->find(array('p_id' => $p_id,'a_id'=>$a_id,'url'=>$url));
        $href_data = $href_curs->getNext();
        if ( isset($href_data['err_status']) && $href_data['err_status'] )
            $ehrfl = 1;
        if ( is_array($href_data) && (!isset($href_data['canonical_present']) || !$href_data['canonical_present'] ) )
            $ehrfl = 1;
    }
    catch(Exception $e){

    }
    return $ehrfl;
}

function getorphanUrls($p_id,$a_id,$mongoDB){
    $eorph = 0;
    try{
        $orph_curs = $mongoDB->orphaned_urls->find(array('p_id' => $p_id,'a_id'=>$a_id));
        $orph_data = $orph_curs->getNext();
        if (isset($orph_data['orp_urls']) && count($orph_data['orp_urls']))
            return $orph_data['orp_urls'];
    }
    catch(Exception $e){

    }
    return $eorph;
}