<?php
# make sure php has sane values for removing session files, because if this script runs often, that can become lots of data
ini_set('session.gc_probability', 1);
ini_set('session.gc_divisor', 100); 
ini_set('session.gc_maxlifetime', 1440);

# we want buffer control
ob_start();
$oh_time = time();

# This contains a collection of functions we're going to use in this program...
if(!function_exists('json_encode')) {
    include_once("components/json/JSON.php");
}
require_once("includes/fallback_functions.php"); 

require_once "components/import/updatefunctions.php";

# HandleInputArguments parses the parameters passed to the script via web or console
HandleInputArguments();

if (!isset($conf) || empty($conf)) { die('missing required parameter'); }

# Let's proceed
include_once("common.inc.php");
$common_time = time() - $oh_time;
$lptime= time();
# These are the files we need to include.
include_once("includes/logparser.inc.php");
include_once("includes/datamanager.php");
include_once("components/import/filehandle.php");
include_once("components/import/import.php");
if (file_exists("includes/blacklist.php")) { include_once("includes/blacklist.php"); } 
//dump($profile);
# Can I help you?
if ($running_from_command_line==false && LoggedIN()==false) { die("unauthorized"); }

//echo (time()-$oh_time) . " sec on line ". __LINE__ . "\n";

# Big data, be prepared (update can be stopped with stop_update.php)
set_time_limit(86400);
$db->Execute("SET SESSION wait_timeout=3600");

if (!$running_from_command_line) {
	# Standard UI mode means we show header/footer and validate that the user is logged in.
    //require_once "top.php";
		
	if ($validUserRequired && (!$session->canUpdateLogs() || CheckSubscriptionOption("Update","Disabled"))) {
		if($session->canUpdateLogs()){
			# Ignore the restriction
		} else if( CheckSubscriptionOption("Update","Enabled") ) {
			# Ignore the restriction
		} else {
			echo json_encode(array("Status" => "Error", "Message" => _NO_PERMISSION ));
        	$profile->SetUpdateStatus('ready');
        	exit();
        }
	}

	if ($session->isAdmin()) {
		OverrideOptions();
	}
	
} else {
	# With command line mode, we *don't* check user security (if they have shell access, then we assume they
	# have access to anything they want and causing security problems here doesn't really do anything for us).
	if (!isset($profile)) {
		$profile = new SiteProfile($conf);
	}
	if (!$profile->profileloaded) {
		die(_CONFIGURATION.' '.$conf.' '._DOESNT_EXIST);
	}
	
    # Does the current profile need to be updated?    
    # make sure the profile version is up to date
	if ($profile->structure_version < CURRENT_PROFILE_STRUCTURE_VERSION) {
		
		include_once "version_check.php";
		updateDataTableForProfile($profile);
		$profile->Load($profile->profilename); // Load the profile again to apply any changes
		//echo (time()-$oh_time) . " sec on line ". __LINE__ . "\n";
	}
    
    # when we are in command line mode, we allow certain profile parameters to be changed at run time
    OverrideOptions();
}
$lptime= time() - $lptime;

# Check if we have a ready status, if not, update the session counter from db, because we cannot trust that it was updated last time
if ($profile->update_status != 'ready') {
	$profile->sessioncounter = GetLastSessionID($profile);
	# Save it in the DB immediately
	$profile->SetInDB("sessioncounter",$profile->sessioncounter);
}

# Check if we need to automatically reset the update running status
if(reset_update_process() === true){
	$profile->SetUpdateStatus('ready');
}

# This will determine if this update job can continue or not, based on it's last status
if (HandleRunningStatus()===false) {
	exit();
}

asCpanelLogaholic();

//echo (time()-$oh_time) . " sec on line ". __LINE__ . "\n";
# Check if our profile is activated or not
if($running_from_command_line == false) {
	# only check activated or not when the user is not an admin
	if (!$session->isAdmin() && $profile->ProfileActiveCheck()==false) {
		die();
	}
} else {
	if($profile->activated == 0){
		echoConsoleSafe("Please activate this profile to update.",true);
		$profile->SetUpdateStatus('ready');
		exit();
	}
}
//echo (time()-$oh_time) . " sec on line ". __LINE__ . "\n";
# Load GeoIP database
include_once("components/geoip/open_geoip.php");
if (isset($gi)) {
	$geo = 1;
}

# just in case
if(empty($mysqltmp)) {
	$mysqltmp = logaholic_dir()."files/";
}
if (@_LOGAHOLIC_EDITION == 4 || LOGAHOLIC_BASE_EDITION == "cPanel Edition") {
	if ($mysqltmp !== "/var/cpanel/userhomes/cpanellogaholic/") {
		$mysqltmp = "/var/cpanel/userhomes/cpanellogaholic/";
	}
}

# if we are collecting data with javascript, create a log file now that we can process
if (@_LOGAHOLIC_EDITION != 4 && LOGAHOLIC_BASE_EDITION != "cPanel Edition") {
	if($profile->trackermode == 1) {		
		include_once "includes/tabletologfile.php";
		$tabtolog = new tabletologfile();
	} 
}

# Startup the data import class
$import = new Import(false);
//echo "manage keys: $manage_keys";
$import->manage_keys = ShouldWeManageKeys($manage_keys);

# We are running an update now, so we set a flag.
$profile->SetUpdateStatus('running');

//echo (time()-$oh_time) . " sec on line ". __LINE__ . "\n";
# Open the Update Progress Log, but first check if the dir exists.
if (!file_exists($profile->datamanagerDir.$profile->profilename)) {	
	mkdir($profile->datamanagerDir.$profile->profilename); 
	set_permissions($profile->datamanagerDir.$profile->profilename);
}
$progress_log = $profile->datamanagerDir.$profile->profilename."/{$profile->profilename}_update_progress.lwa.log";

# If the file already exists and we want to make a new file for each update then rename the file.. all the checks on this file have already been done.
if(file_exists($progress_log) && !empty($profile->other_settings['backup_update_progress_logs'])){
	$date = date("Ymd_H-i-s", filemtime($progress_log) );
	rename($progress_log, $profile->datamanagerDir.$profile->profilename."/{$profile->profilename}_". $date ."_update_progress.lwa.log");
}

# truncate the old file because we want to start clean..
if(file_exists($progress_log)){
	unlink($progress_log);
}

# Changed the method to a because when we use w we wont get error output because it overwrites the line.
$updatelog = fopen($progress_log, "a");
set_permissions($progress_log);

if ($import->manage_keys == true) {
	echoConsoleSafe("manage keys is true\n",true);
} else {
	echoConsoleSafe("manage keys is false\n",true);
}

# give some feedback to the user before we really start work
lgflush();
if (!$running_from_command_line) {
		//echoNotice(_UPDATE_RUNNING_INFO, 'margin: 5px 10px;');
		if ($session->canUpdateLogs()) {
			//echoNotice(_VIEW_UPDATE_PROGRESS, 'margin: 5px 10px;');
		}
	} else {
		echo "\n ". _PROGRESS .": tail -f ".trim($progress_log). "\n";
	}
lgflush();

# From this point on we want to be able to continue while the user is gone. (update can be stopped with stop_update.php)
ignore_user_abort(true);


if ($profile->trackermode == 2) {
	if (LOGAHOLIC_BASE_EDITION == "cPanel Edition") {
		# not supported in Cpanel
		die("Feature not supported in Cpanel");
	} else if( file_exists("includes/FTP_log_class.php") ){
		require_once 'includes/FTP_log_class.php';
		
		# we have to FTP the file first
		$updir = new UploadDir();
		
		# temp overwrite ?
		$profile->original["logfilefullpath"] =	$profile->logfilefullpath;

		$profile->logfilefullpath = $updir->getUploadDir().$profile->profilename ;
		if(!is_dir($profile->logfilefullpath)){
			mkdir($profile->logfilefullpath);
		}

		if(substr($profile->logfilefullpath,-1) != "/"){
			$profile->logfilefullpath .= '/';
		}
		$ftp_time = time();
		echoConsoleSafe("Starting FTP download.<br>\n",true);
		echoConsoleSafe("PHP using ".memory_get_usage(), true);
		$FTP_class = new FTP_log($profile->ftpserver, $profile->ftpuser, $profile->ftppasswd, $profile->ftpfullpath, $profile->splitfilter, $profile->splitfilternegative, $profile->logfilefullpath, $import->getStartTimeForProfile(), $profile->ftpport, $profile->ftpmode);
		$ftp_time = time() - $ftp_time;
		echoConsoleSafe("FTP connect took $ftp_time",true);
		$ftp_time= time();
		$FTP_class->Files_Download();
		$ftp_time = time() - $ftp_time;
		echoConsoleSafe("FTP download took $ftp_time",true);
		
		lgflush();
		
		# if we are doing this we always have to be in multi file mode, so set that now, just in case
		$profile->original["splitlogs"] =	$profile->splitlogs; $profile->splitlogs = 1;
		
		# when we've ftp'd we know we don't have to do any more filtering on the files, since that has already been done. So, reset the filters
		$profile->original["splitfilter"] =	$profile->splitfilter; $profile->splitfilter = "";
		$profile->original["splitfilternegative"] =	$profile->splitfilternegative; $profile->splitfilternegative = "";
	}
}

# Pre check to see if the path needs a '/'
if(is_dir($profile->logfilefullpath) && substr($profile->logfilefullpath,-1) != "/"){
	$profile->logfilefullpath .= '/';
}

# Check if we have move to done function activated.
if($movedone === true && !file_exists($profile->logfilefullpath.'done/')){
	mkdir($profile->logfilefullpath.'done');
	if (!file_exists($profile->logfilefullpath.'done/')){
		echo json_encode(array(
			"Status" => "Error", 
			"Message" => "We are unable to find the '<b>done</b>' directory at the following location:<br/><ul><li>{$profile->logfilefullpath}</li></ul>Please create a '<b>done</b>'  directory in your logfile directory <b>({$profile->logfilefullpath})</b><br/>We need this to place the completed logfiles in <b>{$profile->logfilefullpath}done/</b>"
		));
		$profile->SetUpdateStatus('ready');
		exit();
	}	
}

# We want to know how long an update takes (log file processing time).
$st_time = time();

if($profile->trackermode == 1) {
	$files[] = "{$mysqltmp}{$profile->profilename}_tracker_log";
	// $delfile = true;
} else {

	if (!file_exists($profile->logfilefullpath)){
		echo json_encode(array(
			"Status" => "Error", 
			"Message" => "Your log file path does not exist:<br/><ul><li>{$profile->logfilefullpath}</li></ul>Please check the settings and change it in Edit Profile > Data Collection</b>"
		));
		$profile->SetUpdateStatus('ready');
		exit();
	}

	# Get the log files in array format
	$files = $import->GetLogFilesArray();
        
}

# record how log it took to get the log files array and calculate the time overhead
$lfdur = time() - $st_time;
$oh_time = time() - $oh_time;
//$oh_mysql_time = $db->total_query_time;


//echo("{$db->total_query_time} ({$import->db->total_query_time})seconds for {$db->num_queries}");
//dump($db->query_log);


# lets get started
$import->LogProcess("We're starting an update job. System time is now ".date("Y-m-d H:i:s",time()), false);
$import->LogProcess("get log filearray: $lfdur - total overhead $oh_time", false);

# If we need to offload log file processing to a remote server, do it now
if (file_exists("Logaholic_offload/offload_logs.php")) {
	include_once "Logaholic_offload/offload_logs.php";
}	

# -------------------------------------------------------------#
# let's set up some variables that we will need during import:
# -------------------------------------------------------------#
	
# no data older than this timestamp will be imported (unless force=true)
$skiptime = $import->getStartTimeForProfile();

$regex_skipips = MakeSkipIpsRegex($profile->skipips);
//die($regex_skipips);
# Create the regex needed to skip files.
$check_skipfiles = explode(",", $profile->skipfiles);
$new_skipfiles = [];

foreach ($check_skipfiles as $skip) {
	$skip = trim($skip);
	if(substr($skip, 0,1)=="!") {
		$only_include = $skip;
		continue;
	}

	# If we have a directory then ignore everything after it..
	if(substr($skip, -1) == "/"){
		$new_skipfiles[] =  str_replace('/', '\/', $skip);
	} else if(strpos($skip, ".") !== false) { # Check if it is an extension 
		$skip = str_replace('/', '\/', $skip);
		$new_skipfiles[] = str_replace('.', '\.', $skip) . "(\?|$)";
	} else {
		$new_skipfiles[] = str_replace('/', '\/', $skip);
	}
}
$regex_skipfiles = implode("|", $new_skipfiles);
//dump($regex_skipfiles);
//exit();
# create the regex to filter url parameters
$regex_urlparamfilter = PrepareUrlParamFilter();

# Create the regex for equivdomains
$regex_equivdomains = PrepareEquivDomainFilter();

# Create the regex for google parameters
$regex_googleparams = str_replace(" ", "", str_replace(",", "|", $profile->googleparams));

# Prepare the important params
$dynamic_pages = [];
if(is_array($profile->importantURLParams)){
	foreach($profile->importantURLParams as $param) {
		$dynamic_pages[$param['filename']] = $param['importantparams'];
	}
}

# set up a clean sessions array
$sessions = [];
$sessioncounter = $profile->sessioncounter;

# let's keep track of the total number of lines we process during this update run
$totlines=0;

# first, check if we need to run datamanager
$dataManager = new DataManager();
$dataManager->print = 3;
if ($dataManager->test_Automanager($profile)!==false) {
	$dm_start = time();
	$import->EnableKeys($profile->tablename, true);
	$dataManager->AutoManager($profile, true);
	$dm = 1;
	$dm_took = time() - $dm_start;
} else {
	if (!empty($dataManager->testfeedback)) {
		$import->LogProcess($dataManager->testfeedback, false);
	}
}
if (count($files) > 0) {
	$import->EnableKeys($profile->tablename, false);
	$import->Createlogtable();
}

# Set the preparsed log file location (The file where Perl/PHP writes to).
$new_log_file = "{$mysqltmp}{$profile->profilename}_lwa_log";
# make sure we don't have an old copy of this file hangin around
if(file_exists($new_log_file)){
	if(!@unlink($new_log_file)){
		echoWarning("Failed to delete file: <b>$file</b>","margin:5px 10px;");
	}
}

# --------------------------------------------------------------------------------------------------------------------- #
# For each log file, we'll parse it, and insert it into the database, among other stuff; keep reading to discover more.
# --------------------------------------------------------------------------------------------------------------------- #

$filecount = count($files);
$totalphptime = 0;
$tot_lf_size = 0;
$f = 0;

foreach($files as $file) {
	$sessions = [];
	$import->StopOrContinue();

	$f++;

	if (!file_exists($file)) {
		$import->LogProcess("$file does not exist");
		continue;
	}

	if (!is_readable($file)) {
		$import->LogProcess(_FILE_IS_NOT_READABLE ." : " . $file);
		continue;
	}

	if (filesize($file)==0) {
		$import->LogProcess("$file is empty");
		$import->setFileHistory($file, "lastmodtime", filemtime($file));
		continue;
	}


	
	# Create a empty array for the bandwidth or rest it for new file
	$import->bandwidth = [];

	# first, check if we need to run datamanager - but only if it makes sense in terms of volume
	if ($import->num_import_lines > 200000) {
		$dataManager = new DataManager();
		$dataManager->print = 3;
		$dataManager->AutoManager($profile);	
	}
	
	if (substr($profile->logfilefullpath.$file, -3, 3) == ".gz") {
		$filehandle = new gzhandle();
	} else {
		$filehandle = new fhandle();
	}
	$purephptime = getmicrotime();
	$logfile = $filehandle->fileopen($file, 'r');
	
	$import->LogProcess("Skiptime is now ".date('Y-m-d H:i:s',$skiptime));
	
	// if (strtoupper(substr(PHP_OS, 0, 3)) !== 'WIN') {
	// 	$load = sys_getloadavg();
	// 	$import->LogProcess("System load {$load[0]} {$load[1]} {$load[2]}");
	// }
	
	$start_file_processing = time();

	# This variable is being set to let the user know which file we're currently updating.
	$human_readable_file = explode("/", $file);
	$human_readable_file = $human_readable_file[(count($human_readable_file) - 1)];
	$import->LogProcess("Start processing $file");
	$bsize = filesize($file);
	$size = round(($bsize / 1024) / 1024, 2);
	$tot_lf_size = $tot_lf_size + $bsize;
	$format = formatOfLogFile($file);
    if (!isset($format['ClassName'])){
        $format['ClassName']='ApacheCombinedCookieLogParser';
    }
	$import->LogProcess(_PARSING_LOG_FILE." ({$size} mb, {$format['ClassName']}) ...", false);
		
	if ($import->setLogParser($format['ClassName'])!==false) {
		$import->log_parser->Initialize($logfile);
		$import->LogProcess($import->log_parser->lasterrormessage);
	} else {
		$import->LogProcess("Unknown log file format");
		continue;
	}
		
	$outputfile = fopen($new_log_file, 'a+');
	set_permissions($new_log_file);
	
	$c = 0;	
	# if we have stored a last log position, start reading the file from that position
	$lastlogpos	= $import->getFileHistory($file, "lastlogpos");
	$firstlogline = $import->getFileHistory($file, "firstlogline");
	
	# check if file is still the same 
	$flogline = $filehandle->filegets($logfile, 0);	
	if ($firstlogline !== md5($flogline)) { 
		$lastlogpos=0;
		$firstlogline = md5($flogline);
	} else {
		//echo "\n ja hoor ik ben gelijk $firstlogline == ".md5($flogline)."\n";
	}
	
	if ($force!==true && !isset($check_file_history)) {
		$filehandle->fileseek($logfile, $lastlogpos);
	}

	# Loop through each line of the log file
	$ll=0;			

	//$import->LogProcess("Start PHPanalyzeLine's");

	while ($logline = $filehandle->filegets($logfile, 0)) {		
		# Convert the log line to our preparsed log format.
		$newline = $import->PHPanalyzeLine($logline);
	
		# Write our newly created log line to our preparsed logfile
		fwrite($outputfile, $newline);
		unset($newline);	
		//lgflush();
		$ll++;
	}
	// if ($ll > 0) {
	// 	$import->LogProcess("crawl: $import->count_crawl, human: $import->count_human");
	// }

	$ppt = getmicrotime() - $purephptime;
	$totalphptime = $totalphptime + $ppt;
	if ($ll > 0 && $ppt > 0) {
		$ppt = ($ppt == 0) ? 0 : round($ll/$ppt);
		$import->LogProcess("Parsing speed was $ppt lines per second");
		$ppts[] = $ppt;
	}
	# Fetch the byte value of the file pointer.
	$lastlogpos = $filehandle->filetell($logfile);
	
	fclose($outputfile);
	
	$took = $import->thisTook($start_file_processing);
	
	$m = memory_get_usage();
	$m =round(($m / 1024) / 1024);
	$t =round(($import->totmem / 1024) / 1024);
	$import->LogProcess("PHP is now using $m MB / {$t} MB");
	
	lgflush();
	
	# If there is no last log position, set it to zero.
	if(!isset($lastlogpos)) {
		$lastlogpos = 0;
	}
	
	# If there is no first log line, make it empty.
	if(!isset($firstlogline)) {
		$firstlogline = '';
	}

	# store the bandwidth in datafiles
	$import->StoreBandwidth();

	# store the Authenticated Visitors in datafiles
	$import->SaveAuthenticatedVisitors();
	
	# Save the first log line and last log position.
	$import->setFileHistory($file, "lastlogpos", $lastlogpos);
	$import->setFileHistory($file, "firstlogline", $firstlogline);
	$import->setFileHistory($file, "lastmodtime", filemtime($file));
	
	$profile->sessioncounter = $sessioncounter;
	
	# it's more efficient to add some more data before doing the rest
	if (($filecount > $f) && filesize($new_log_file) < 20971520)  {
		$import->LogProcess("Aggregating log lines to tmp file for more efficient import");
		$filehandle->fileclose($logfile);
		continue;
	}

	if ($ll > 1) {
		# Do all the import to database queries
		ImportLogToDatabase($new_log_file);
	}

	# We are using the javascript tracker so drop the tracker table.
	if($profile->trackermode == 1) {
		$db->Execute("DELETE FROM ". $profile->tablename_trackerlog);
	}
	
	# Remove the tracker log file..
	if($profile->trackermode == 1) {
		if(!@unlink("{$mysqltmp}{$profile->profilename}_tracker_log")){
			echoWarning("Failed to delete file: <b>$file</b>","margin:5px 10px;");
		}
	}
	
	# if the $delfile parameter  is true delete the original log file we just processed
	if ($delfile == true) {
		if(!@unlink($file)){
			echoWarning("Failed to delete file: <b>$file</b>","margin:5px 10px;");
		}
	}
	
	# print some information about this job
	$import->LogProcess("Finished processing {$human_readable_file}", false);
	$tottime = time() - $start_file_processing;
	$took = $import->thisTook($start_file_processing);
	if ($tottime > 0) {
		$speed = $import->num_import_lines / $tottime;
	} else {
		$speed = $import->num_import_lines;
	}
	$totlines= $totlines + $import->num_import_lines;
	//let's turn off log duration for now and save some time, we really already have everything via the regular log
	//$import->LogDuration("Done!", $speed, $tottime);
	$speed  = round($speed);
	$import->LogProcess("Imported {$import->num_import_lines}, Speed was $speed lines per second");
	$import->LogProcess("---------------------------------------------------------", false);
	
	#close the filehandler
	$filehandle->fileclose($logfile);
	
}

# If the file still exists it is because the script had skipped the last log while there still is data to import
if(file_exists($new_log_file)){
	$import->LogProcess("---------------------------------------------------------", false);
	$import->LogProcess("Start import Aggregated log lines from tmp file");
	# Do all the import to database queries
	ImportLogToDatabase($new_log_file);

	# print some information about this job
	$import->LogProcess("Finished processing {$human_readable_file}", false);
	
	$tottime = time() - $start_file_processing;
	$took = $import->thisTook($start_file_processing);
	if ($tottime > 0) {
		$speed = $import->num_import_lines / $tottime;
	} else {
		$speed = $import->num_import_lines;
	}
	$totlines= $totlines + $import->num_import_lines;
	//let's turn off log duration for now and save some time, we really already have everything via the regular log
	//$import->LogDuration("Done!", $speed, $tottime);
	$import->LogProcess("Imported {$import->num_import_lines}, Speed was $speed lines per second");
	$import->LogProcess("---------------------------------------------------------", false);
}
$human_readable_file = "";

# move logs to done folder ?
if($movedone === true){
	MoveDone($files);
}

if (count($files) > 0) {
	# Turn the keys back on. Enable keys is done twice to prevent some mysql error we got. Keep it in.
	$import->LogProcess("Rebuilding index on main table...", false);
	$import->EnableKeys($profile->tablename, true);
	$import->EnableKeys($profile->tablename, true);
}

$took = $import->thisTook($took);
$import->LogProcess(_FINALIZING_UPDATE."...", false);


$maxrange = GetMaxDateRange($profile);

# check which timestamp has been last inserted
if($import->latest_timestamp > 0){
	$last_inserted_time = $import->latest_timestamp;
} else {
	$last_inserted_time = $maxrange['to'];	
}

# merge our existing updatehistory with the new affected dates array
// clearstatcache();
// $fmt = filemtime($progress_log); // use the timestamp from the progress log so we are sure we are using the system time ffor comparisons
// $profile->updatehistory[$fmt] = array_keys($import->affected_dates);
// dump($profile->updatehistory);
// $profile->updatehistory = CleanupHistory();

# if force is true, we have to delete the affected data files now (because we don't know what is going wrong with the CleanupHistory thing if force=true)
// if ($force === true) {
// 	foreach($import->affected_dates as $adate => $v) {
// 		if (strlen($adate) <= 6) {
// 			continue;
// 		}
// 		$startof_day = mktime(0,0,0,substr($adate,4,-2),substr($adate,6,7),substr($adate,0,4));
// 		$endof_day = mktime(23,59,59,substr($adate,4,-2),substr($adate,6,7),substr($adate,0,4));
// 		DeleteDataFiles($profile,$startof_day,$endof_day);
// 		//echo "Deleting for $adate<br>\n";
// 	}
// }
// // if ($import->oldest_timestamp > 0) {
// // 	DeleteDataFiles($profile,$import->oldest_timestamp,$last_inserted_time);
// // }

# store the max timestamp
$profile->min_db_timestamp = $maxrange['from'];
$profile->max_db_timestamp = $maxrange['to'];

# Do update summaries for all the data we have just imported.
$update_sum_time = time();
if (!isset($ignoreReports) || $ignoreReports !== 1) {
	if ($totlines> 1500) {
		UpdateDatafiles($last_inserted_time);
	}
}
$update_sum_time = time() - $update_sum_time;

# store the first timestamp ever found (not just db, datafiles too)
if($profile->firstRequestTimeEver == 0) {
	$profile->firstRequestTimeEver = LogaholicFirstTimestamp();
}

# get a total pageviews for the last 30 days
$profile->thirtyday_pageviews = GetThirtydayPageviews($profile);

//exit();
# We are done with update; unflag it.
$profile->SetUpdateStatus('ready');

# now it's time to check if we need to send any email alerts
include "includes/emailalerts.php";
$e = new EmailAlerts();
$email_feedback = $e->SendAlerts();
if ($e->send > 0) {
    $email_feedback[] = "\n\n<br>".str_replace('%a',$e->send,_SENT_X_ALERTS)."<br>";
}

# now calculate how long everything took
$took = $import->took($st_time);
$totaltook = $import->sec2time($took);
if (!isset($dm_took)) {
	$dm_took =0;	
}

$net_time = $took - $dm_took;

if ($net_time > 0) {
	$lps = round($totlines/$net_time);
} else {
	$lps = $totlines;
}
if (isset($dm)) {
	$dm = "(with datamanager)";
} else { $dm = ""; }

$plps = 0;
if (!empty($ppts)) {
	$plps = (array_sum($ppts) / count($ppts));
}

# print some stats about this update job
/*
$peak = (memory_get_peak_usage()/1024)/1024;
echoWarning($peak);

$arr = get_defined_vars();
$peak = (memory_get_peak_usage()/1024)/1024;
echoWarning(count($arr));
echo "strlen of all vars:".strlen(serialize($arr));
function memhog($name, $arr) {
	$memhog = [];
	$i=0;
	$n = count($arr);
	foreach ($arr as $k => $v) {
		if ($k=='GLOBALS') { continue; }
		$memhog[$k] = strlen(serialize($v));
		//echoWarning("strlen of $k:".$memhog[$k]);
		$i++;
	}
	arsort($memhog);
	echo "<h3>$name has $n items</h3><table>";
	foreach($memhog as $k => $v) {
		echo "<tr><td>$k</td><td>".number_format($v)."</td></tr>";
	}
	echo "<tr><td>Total rows</td><td>".$i."</td></tr>";
	echo "</table>";
}

memhog("all", $arr);

memhog("ua_parser", $arr['ua_parser']);

echoWarning("sessions array has ".count($arr['sessions'])." elements" );
//dump($arr['ua_parser']);

//memhog("globals", $arr['GLOBALS']);
*/

$statsline = "[Stats]; LFBytes; $tot_lf_size; LFLines; $totlines; LPS; $lps; NETPTime; $net_time; TOH; $oh_time; FOH; $lfdur; SumT; $update_sum_time; DMTime; $dm_took; TotTime; $totaltook; ({$profile->profilename}); CTIME; $common_time;  PLPS; $plps; TOTPHPT; $totalphptime; ";
if ($running_from_command_line===true) {
	#I want my message in english
	$lastline = "[Finished] Processed $totlines lines ($lps/lps) for {$profile->profilename}. ".str_replace("%s1", $took, _DONE_UPDATE_TOOK_SECONDS." (".$totaltook.")");
} else {
	$lastline = str_replace('%profile%', $profile->profilename, _PROCESSED_LINES_FOR);
	$lastline = str_replace('%lines%', "$totlines ($lps/lps)", $lastline);
	$lastline = "[". _FINISHED ."] $lastline. ".str_replace("%s1", $took, _DONE_UPDATE_TOOK_SECONDS." (".$totaltook.")");
}

# restore the default settings because of overwrite settings..
restoreOriginalOptions();

$profile->last_update_finished = time();
$profile->Save();

# echo the last line in the progress log
$import->LogProcess($statsline);
$import->LogProcess($lastline);

//lgflush();
fclose($updatelog);
$updatelog = '';

# and send it to the user
if ($running_from_command_line) {
	echo "\nFinish: cat ".trim($progress_log). "\n";
	echo $statsline."\n";
	echo $lastline."\n";

} else {
	$message = "<strong>".$lastline."</strong>";

	# using force will disregard all date/time checking during import, so warn the user
	if ($force) {
		$message = Warning(_FORCING_IMPORT) . $message;
	}

	# No geo lite city .dat found warn the user
	if(empty($geo)){
		$message = Warning(_NO_GEO_DATA) . $message;
	}

	

	$updatedata = array(
		"profilename" => $profile->profilename,
		"LFBytes" => $tot_lf_size,
		"LFLines" => $totlines,
		"LPS" => $lps,
		"NETPTime" => $net_time,
		"TOH" => $oh_time,
		"FOH" => $lfdur,
		"SumT" => $update_sum_time,
		"DMTime" => $dm_took,
		"TotTime" => $totaltook,
		"CTIME" => $common_time,
		//"OHSQL" => $oh_mysql_time,
		//"TOTSQL" => $db->total_query_time,
		//"NUMQ" => $db->num_queries,
		"PLPS" => $plps,
		"TOTPHPT" => $totalphptime,
		"TOTF" => count($files),
	);

	echo json_encode(array("Status" => "Success", "Message" => "<strong>".$message."</strong>", "email_feedback" => $email_feedback, "Data" => $updatedata) ).PHP_EOL;
}
?>