<?php

/**
* @desc This is the master report class. All other reports should be childen of this parent class
* it is assumed that this is included somewhere where common.inc.php is also included 
*/
if(!defined('APP_INCLUDE')){ die('invalid inclusion'); }

include_once(logaholic_dir()."includes/logparser.inc.php");

include_once(logaholic_dir()."components/import/import.php");

Class DataManager {
	var $print;
	var $set_time_limit;
	
	function __construct() {
		$this->print = 2;
		$this->set_time_limit = 86400;		
		
	}
	
	function test_Automanager($profile, $run=false) {
		global $db,$force,$skiptime;
		
		$this->testfeedback = "";

		if($force === true){
			$this->testfeedback = "Don't run datamanager because force=true";
			return false;
		}
		
		$setting = $profile->GetOtherSettings("datamanagersettings",getGlobalSetting("datamanagersettings",0));
		if($setting == 0) {
			return false;
		}else {			
			$q = $db->Execute("SELECT MIN(timestamp) as mindate, MAX(timestamp) as maxdate, count(*) as hits FROM {$profile->tablename}");
			$data = $q->FetchRow();
			$max = $data["maxdate"];
			$min = $data["mindate"];
			$hits = $data["hits"];
			
			$del_to = mktime(0,0,0,date("m",$max)-$setting,01,date("Y",$max)) - 1;
			
			if(empty($min) || empty($del_to)){
				return false;
			}
			
			# we will only start datamanager if
			# 1. the daterange exceeds the period set for datamanager AND
			# 2. there are more than 300000 records in the table AND
			# 3. the ramndomizer allows it (so all datamanagers don't start on the same updateall run)
			
			if( $min < $del_to ){
				// echo "AutoManager says datamanager would have run ( $	min < $del_to )";
				# running datamanager for less than this number of records is not worth it
				if ($hits > 100000) {
					if ($run===true) {
						set_time_limit($this->set_time_limit);
						return $setting;
					} else {
						# do you feel lucky, punk?
						if (rand(1,4) == 1) {
							set_time_limit($this->set_time_limit);
							return $setting;
						} else {
							$this->testfeedback = "Don't run datamanager right now due to randomized start";
						}
					}
				} else {
					$this->testfeedback = "Don't run datamanager because hits < 300000";
				}				
			} 
			return false;
		}
	}

	function AutoManager($profile, $run=false){
		global $db,$force,$skiptime;

		$setting = $this->test_AutoManager($profile, $run);
		if ($setting!==false) {			
			$this->ClearDatabase($profile, $setting);
			$this->cleanLogStorage($profile);
		}
	}
	
	function createReportDataFiles($from,$to){
		global $reports;
		
		if( date("Y m d",$to) ==  date("Y m d",time())) {
			$to = mktime(23,59,59,date("m",$to),date("d",$to)-1,date("Y",$to));	
		}
		
		foreach($reports as $k => $v){
			$start = time();
			if($v['Distribution'] == 'Unlisted') {
				continue;
			}
			if(isset($v["Update"]) && $v["Update"] === false) {
				continue;
			}

			$r = new $reports[$k]["ClassName"]();
			$r->label = constant($k);		
			$r->UpdateStats($from,$to,2);	
			//$this->LogProcess("Memory usage is now ". memory_get_usage());
		}
	}
	
	function getTables() {
		global $profile;
		
		# Get an array containing all needed tables for the current profile.
		$tables = [];
		$tables[] = array('table' => $profile->tablename_urls , 'field' => 'url', 'sfield' => 's.url');
		$tables[] = array('table' => $profile->tablename_urlparams , 'field' => 'params', 'sfield' => 's.params');
		$tables[] = array('table' => $profile->tablename_referrers , 'field' => 'referrer', 'sfield' => 's.referrer');
		$tables[] = array('table' => $profile->tablename_refparams , 'field' => 'params', 'sfield' => 's.refparams');
		$tables[] = array('table' => $profile->tablename_keywords , 'field' => 'keywords', 'sfield' => 's.keywords');
		$tables[] = array('table' => $profile->tablename_useragents , 'field' => 'useragent', 'sfield' => 's.useragentid');
		$tables[] = array('table' => $profile->tablename_visitorids , 'field' => 'visitorid', 'sfield' => "s.visitorid");
		
		return $tables;
	}
	
	function took($start) {				
		$secs = time() - $start;		
		$hours = str_pad(floor($secs / (60 * 60)),2,'0',STR_PAD_LEFT);
		$divisor_for_minutes = $secs % (60 * 60);
		$minutes = str_pad(floor($divisor_for_minutes / 60),2,'0',STR_PAD_LEFT);
		$divisor_for_seconds = $divisor_for_minutes % 60;
		$seconds = str_pad(ceil($divisor_for_seconds),2,'0',STR_PAD_LEFT);
		return "{$hours}:{$minutes}:{$seconds} ($secs secs)";
	}
	
	function ClearDatabase($profile,$months = 0, $forcerun = false, $print = 2) {
		global $db,$DatabaseName, $validUserRequired, $session, $skiptime;
		$this->print = $print;
		# No database Limit set, so nothing happens to the database.
		if($months == 0){
			$this->LogProcess("Setting is set to 'No Limit'. Datamanager does not need to clear the database..");
			return false;
		}

		set_time_limit(0);

		$start_time=time();

		if($this->print != 'stream'){
			ob_start();
		}
		
		$q = $db->Execute("SELECT MIN(timestamp) as mindate, MAX(timestamp) as maxdate FROM {$profile->tablename}");
		$data = $q->FetchRow();
		$max = $data["maxdate"];
		$min = $data["mindate"];	

		# Set the timestamp to where the data must be deleted.
		$del_to = mktime(0,0,0,date("m",$max)-$months,01,date("Y",$max)) - 1;
		$range = $max - $del_to;	
		
		if($forcerun == false){
			if ($min >= $del_to) {
					$this->LogProcess("ClearDatabase says datamanager has nothing to do ( $min < $del_to )");
					return;
			} else {
					$this->LogProcess( "ClearDatabase says datamanager will run now ( $min < $del_to )");
			}
		}
		
		$time = mktime(0,0,0,date("m",$min),date("d",$min),date("Y",$min));
		$export = [];
		while($time < $del_to){
			if (!file_exists("{$profile->datamanagerDir}{$profile->profilename}/log/". date("Ymd",$time).".gz")) {
				$export[] = $time;
			}			
			$time = mktime(23,59,59,date("m",$time),date("d",$time),date("Y",$time)) + 1;
		}
		
		foreach($export as $k => $v ){
			$f = mktime(0,0,0,date("m",$v),date("d",$v),date("Y",$v));
			$t = mktime(23,59,59,date("m",$v),date("d",$v),date("Y",$v));
			$this->Export($profile, $f, $t, 1);
			$this->StopOrContinue();
		}		
		
		#export DataFiles
		$start = time();
		$this->createReportDataFiles($min,$del_to);
		$this->LogProcess( "<br/> CREATING DATA FILES TOOK: ". $this->took($start) . "<br/>");
		$this->StopOrContinue();
		#The main table
		
		#create Temp Main Table	
		$this->LogProcess( "Duplicating table {$profile->tablename}...");
		

		$db->Execute("DROP TABLE IF EXISTS {$profile->tablename}_temp");
		createMainProfileTable($profile->tablename."_temp");
		
		$db->Execute("DROP TABLE IF EXISTS {$profile->tablename_crawl}_temp");
		createMainProfileTable($profile->tablename_crawl."_temp");
		
		$main_tables[] = $profile->tablename;
		$main_tables[] = $profile->tablename_crawl;
		
		# this is what we have to do for the  main and crawl tables
		foreach($main_tables as $table) {		
			$select = "SELECT * FROM {$table} WHERE timestamp >= $del_to";		
			$insert = "INSERT INTO {$table}_temp {$select}";
			$drop = "DROP TABLE IF EXISTS {$table}";
			$rename = "RENAME TABLE {$table}_temp TO {$table}";
			
			# Insert Data into temp Main Table
			# disable indexes on the copy so our insert is faster
			$db->Execute("ALTER TABLE {$table}_temp DISABLE KEYS");
			$this->LogProcess( "Inserting data into {$table}_temp..");
			
			$start = time();
			$db->Execute($insert);
			$rows = $db->Affected_Rows();
			$db->Execute("ALTER TABLE {$table}_temp ENABLE KEYS");
			$this->LogProcess( "Took: " . $this->took($start) . "</br>");
			
			#Return the amount of rows affected		
			$this->LogProcess( "Affected $rows rows from {$table}.<br/>");
			
			
			# Delete the old profile table
			$this->LogProcess( "Dropping  old table {$table}..");
								
			$db->Execute($drop);			
			
			# Rename the temp table to the original name
			$this->LogProcess( "Renaming {$table}_temp to {$table}..");
					
			$db->Execute($rename);	
			
			# sometimes the table crashes (seen in only one case, but persistent), so let's do a rapair table just to be sure
			$db->Execute("REPAIR TABLE $table");
		}
		
		$tables = $this->getTables();
		# this is what we do for all the subtables		
		foreach($tables as $table){			
			# set up all the queries we need to do for each table			
			$dropoldtemp = "DROP TABLE IF EXISTS {$table["table"]}_temp";
			$create = "CREATE OR REPLACE TABLE  {$table["table"]}_temp LIKE {$table["table"]}";
			$select =  "SELECT DISTINCT t.* FROM {$table["table"]} as t, {$profile->tablename_merge} as s WHERE t.id = {$table["sfield"]}";
			$insert = "INSERT INTO {$table["table"]}_temp {$select}";
			$drop = "DROP TABLE IF EXISTS {$table["table"]}";
			$rename = "RENAME TABLE {$table["table"]}_temp TO {$table["table"]}";
									
			# drop old temp table if it exists, and create a new one
			$db->Execute($dropoldtemp);
			$this->LogProcess( "Duplicating table {$table["table"]}...");
			
			$db->Execute($create);

			# disable indexes on the copy so our insert is faster
			$db->Execute("ALTER TABLE {$table["table"]}_temp DISABLE KEYS");
		
			# insert the data we want to keep into the temp table
			$this->LogProcess( "Inserting data into  {$table["table"]}_temp..");
				
			$start = time();		
			$db->Execute($insert);			
			$rows = $db->Affected_Rows();
			$this->LogProcess( "Affected $rows rows from {$table["table"]}<br />");
			
			# enable the keys
			$db->Execute("ALTER TABLE {$table["table"]}_temp ENABLE KEYS");			
			$this->LogProcess( "Took: " . $this->took($start) . "<br />");			
			
			
			# delete the original table
			$this->LogProcess( "Dropping  old table {$table["table"]}..");
						
			$db->Execute($drop);
			
			# rename temp to the original name
			$this->LogProcess( "Renaming {$table["table"]}_temp to {$table["table"]}..");
						
			$db->Execute($rename);	
		}
		
		# the conversions table is a copy of info in main table so we can just delete the date range
		$db->Execute("DELETE FROM $profile->tablename_conversions WHERE timestamp < $del_to");
				
		# we're done
		$this->LogProcess( "Finished Clearing Database. The whole process took ".$this->took($start_time)."<br />");
	}
	
	function CreateTmpTxtFileFromGzFile($path,$file){
		
		# Write the data in the temp txt file.
		$fp = fopen($path.$file.'.txt', 'w+');
		
		$gz = gzopen($path.$file.".gz", 'r');
		while (!gzeof($gz)) {
		   $contents = gzgets($gz);
			fwrite($fp, $contents);
		}
		gzclose ($gz);
				
		fclose($fp);		
		
		return $file. ".txt";
	}
	
	function Import($profile,$from,$to,$print = 1){
		global $db, $vnum , $databasedriver, $reports;
		
		set_time_limit(0);
		
		if($print != 0 || $print == 'stream'){	
			$this->LogProcess( "Start Import: <b>". date("d M Y",$from) ." - ". date("d M Y",$to) ."</b><br/>");
		}
		
		$basepath = "{$profile->datamanagerDir}{$profile->profilename}/log/";
		$time =  mktime(00, 00, 00,date("m",$from), date("d",$from), date("Y",$from));
		$oriTo = $to;

		$importer = new Import();
		$importer->print = $print;

		while($time <= $oriTo) {

			$importer->oldest_timestamp = mktime(0,0,0,date("m",$time),date("d",$time),date("Y",$time));
			
			$to = mktime(23,59,59,date("m",$time),date("d",$time),date("Y",$time));
			
			$log_file = date("Ymd",$time);

			$f = "{$basepath}{$log_file}.gz";

			if(!file_exists($f)){

				if($print != 0 || $print == 'stream'){	
					$this->LogProcess( "$f does not exist..");	
				}	

				$time = $to + 1;				
				continue;
			}
				
			$new_log_file = $this->CreateTmpTxtFileFromGzFile($basepath,$log_file); 
			$mem = (memory_get_usage() /1024)/1024;
			// echo "PHP Memory now: {$mem}<br/>"; 

			if($print != 0){
				$this->LogProcess( "Importing: <b>". date("d M Y",$time) ."</b><br/>");
			}			
			
			# Import the data in MySQL
			$importer->ImportLog("{$basepath}{$new_log_file}");
			
			# Insert the data in the corresponding tables.
			$importer->insertIDs();
			
			# Insert the visitor ID's
			$importer->insertVisitorIDs();
			
			# Insert the ID's in the main table.
			$importer->insertNormalized();
			
			# Add internal referrer detection.
			$importer->UpdateInternalReferrers();
			
			# Add useragent detection
			$importer->UpdateUseragents();
			
			# Move the crawler traffic to a table of it's own
			$importer->separateBots();
			
			# Copy the conversions to a table of it's own
			$importer->copyConversions();			
			
			# We're done with our log table; delete it.
			$db->Execute("TRUNCATE table ". $profile->tablename ."log");

			if(file_exists("{$basepath}{$new_log_file}")){
				unlink("{$basepath}{$new_log_file}");
			}
			
			if($print != 0 || $print == 'stream'){	
				$this->LogProcess( "<br/>");
			}
			
			$time = $to + 1;
		}	

		if($print != 0 || $print == 'stream'){	
			$this->LogProcess( "Finished Importing..");	
		}
		$this->UpdateMaxDbTimestamp($profile);	
	}	

	function UpdateMaxDbTimestamp($profile){
		global $db;
		$r = $db->Execute("SELECT MAX(timestamp), MIN(timestamp) from ". $profile->tablename_merge);
		$row = $r->fetchrow();
		$profile->max_db_timestamp = $row[0];
		$profile->min_db_timestamp = $row[1];
		$profile->Save();
	}
	


	function Export($profile, $from, $to,$print = 0){
		global $db;
		# Export the .gzipped backup file.
		//set_time_limit(0);

		if($print != 0 || $print == 'stream'){	
			$this->LogProcess( "Start Export: <b>". date("d M Y",$from) ." - ". date("d M Y",$to) ."</b><br/>");
		}

		
		$q = $db->Execute("SELECT MIN(timestamp) as mindate, MAX(timestamp) as maxdate FROM {$profile->tablename}");
		$data = $q->FetchRow();
		if ($from < $data["mindate"] && $to < $data["mindate"] || $from > $data["maxdate"]) {
			$this->LogProcess("The date range you chose is not available in the database. Database data spans from ".date("Y-m-d",$data["mindate"]) . " to ".date("Y-m-d",$data["maxdate"]));
			return;
		}
		
		if($from < $data["mindate"]){
			$from = $data["mindate"];
		}		
		if($to > $data["maxdate"]){
			$to = $data["maxdate"];
		}
		
		$time =  mktime(00, 00, 00,date("m",$from), date("d",$from), date("Y",$from)); 
		$oriTo = $to;
		while($time <= $oriTo) {
			
			$to = mktime(23,59,59,date("m",$time),date("d",$time),date("Y",$time));
			# get all data from the database within the daterange
			$query = "SELECT a.timestamp, 
						v.ipnumber AS ipnumber, 
						u.url AS url, 
						up.params AS params, 
						a.status, a.bytes, 
						r.referrer AS referrer,
						k.keywords as keywords,
						rp.params AS refparams, 
						b.useragent AS useragent,
						v.visitorid as visitorid,
						a.sessionid as sessionid,
						a.crawl as crawl,
						a.country as country

					FROM {$profile->tablename_merge} as a, 
						{$profile->tablename_visitorids} AS v, 
						{$profile->tablename_urls} AS u, 
						{$profile->tablename_urlparams} AS up, 
						{$profile->tablename_referrers} AS r,
						{$profile->tablename_keywords} as k,
						{$profile->tablename_refparams} AS rp, 
						{$profile->tablename_useragents} AS b 

					WHERE a.timestamp >= {$time} 
						and a.timestamp <= {$to} 
						AND (a.visitorid = v.id
						AND a.url = u.id 
						AND a.params = up.id 
						AND a.referrer = r.id
						and a.keywords = k.id
						AND a.refparams = rp.id 
						AND a.useragentid = b.id)

					ORDER BY a.timestamp";
			
			$bupfile = "{$profile->datamanagerDir}{$profile->profilename}/log/". date("Ymd",$time).".gz";
			
			$dir_1 = "{$profile->datamanagerDir}{$profile->profilename}";
			$dir_2 = $dir_1."/log/";

			if(!file_exists($bupfile)) {
				if(!file_exists($dir_1)) {
					mkdir($dir_1);
					set_permissions($dir_1);
				}
				if(!file_exists($dir_2)) {
					mkdir($dir_2);
					set_permissions($dir_2);
				}
			} else {
				# if the file exists, we'll assume we only need to regenerate it if it is the last file, so check if the next file exists
				$test = "{$profile->datamanagerDir}{$profile->profilename}/log/". date("Ymd",($to+1)).".gz";
				if(file_exists($test)) {
					$time = $to + 1;
					//echo "skipping $bupfile<br>";
					continue;
				}
			}

						
			$fp = gzopen ($bupfile,"w");
			$buffer = "";
			
			$q = $db->Execute($query);
			while ($data=$q->FetchRow()) {
				if ($data["refparams"]=="?") {
					$data["refparams"]="-";
				}				
				
				$buffer = implode("|LWA|", array(
					$data["ipnumber"],
					"-",
					"-",
					$data["timestamp"],
					"",
					$data["url"] ,
					$data["status"],
					$data["bytes"],
					$data["referrer"],
					$data["useragent"],
					md5($data["useragent"]),
					"",
					$data["params"],			
					$data["refparams"],
					$data["keywords"],
					$data["visitorid"],
					$data["country"],
					$data["crawl"],
					$data["sessionid"]));
				
				$buffer = str_ireplace(array("\r", "\n", "%0a", "%0d"), '', $buffer);
				gzwrite ($fp, $buffer);
				gzwrite ($fp, "\n");
				$buffer=NULL;
			}
			gzclose ($fp);
			set_permissions($bupfile);

			$this->LogProcess("wrote file: $bupfile<br>");	
				
			
			$time = $to + 1;
		}

		if($print != 0 || $print == 'stream'){	
			$this->LogProcess( "<br/>Finished Export..");	
		}	
	}
	
	# This function prints stuff to the update progress log.
	function LogProcess($message) {
		global $updatelog, $running_from_command_line;
		
		if($this->print == 'stream'){
			echo "data: ". $message . "\n\n";
			//ob_flush();
			//flush();
		}else if(!empty($updatelog)) {
			fwrite($updatelog, $message."\n");
		} else {
			echoConsoleSafe($message."<br/>");
		}
		// flush it!
		lgflush();			
		
	}
	
	function cleanLogStorage($profile){
		$bytes = $profile->datamanagerMaxStorage * 1024 * 1024;
	
		$dir = $profile->datamanagerDir.$profile->profilename."/log";
		if(!is_dir($dir)) {
			return;
		}
		$logs = [];
		$total = 0;
		
		foreach(glob($dir . '/*') as $logfile) {
			$date = explode("log/",str_replace(".gz", "",$logfile));
			$date = $date[1];
			$logs[$date]["file"] = $logfile;
			$logs[$date]["size"] = filesize($logfile);
			$total = $total + filesize($logfile);
		}		
		ksort($logs);
		
		if($bytes > $total) {
			return;
		}
		
		foreach($logs as $log){
			if($bytes > $total) {
				break;
			}
			unlink($log["file"]);
			$total = $total - $log["size"];
			
		}
		return;
	}
	
	
	# This function will stop the update process if a user has remotely turned on the stop flag (via UI)
	function StopOrContinue() {
		global $profile, $sessioncounter,$running_from_command_line;

		if ($running_from_command_line === true) {
			# try this to see if it speeds things up on hostpoint
			return true;
		}

		if($profile->GetUpdateStatus() == "stop"){
			$this->LogProcess("Error: user stopped process", false);
			$profile->SetUpdateStatus('ready');
			$profile->SetInDB("sessioncounter",$sessioncounter);
			die();
		} else {
			return true;
		}
	}
}
?>