commit c8345b135478e6b1ebeead04e9add680bce1ea05 Author: Jonathon Fowler Date: Wed Nov 10 17:05:27 2010 +1000 stats processing performance changes to help Oracle deal with many millions of log table rows diff --git a/admin/statscron.php b/admin/statscron.php new file mode 100644 index 0000000..cd3e22e --- /dev/null +++ b/admin/statscron.php @@ -0,0 +1,134 @@ +disablestatsprocessing = true;" to the config.php. +/// +/// This file is best run from cron on the host system (ie outside PHP). +/// The script can either be invoked via the web server or via a standalone +/// version of PHP compiled for CGI. +/// +/// eg wget -q -O /dev/null 'http://moodle.somewhere.edu/admin/statscron.php' +/// or php /web/moodle/admin/statscron.php + set_time_limit(0); + $starttime = microtime(); + +/// The following is a hack necessary to allow this script to work well +/// from the command line. + + define('FULLME', 'cron'); + + +/// Do not set moodle cookie because we do not need it here, it is better to emulate session + $nomoodlecookie = true; + +/// The current directory in PHP version 4.3.0 and above isn't necessarily the +/// directory of the script when run from the command line. The require_once() +/// would fail, so we'll have to chdir() + + if (!isset($_SERVER['REMOTE_ADDR']) && isset($_SERVER['argv'][0])) { + chdir(dirname($_SERVER['argv'][0])); + } + + require_once(dirname(__FILE__) . '/../config.php'); + require_once($CFG->libdir.'/adminlib.php'); + require_once($CFG->libdir.'/gradelib.php'); + +/// Extra debugging (set in config.php) + if (!empty($CFG->showcronsql)) { + $db->debug = true; + } + if (!empty($CFG->showcrondebugging)) { + $CFG->debug = DEBUG_DEVELOPER; + $CFG->displaydebug = true; + } + +/// extra safety + @session_write_close(); + +/// check if execution allowed + if (isset($_SERVER['REMOTE_ADDR'])) { // if the script is accessed via the web. + if (!empty($CFG->cronclionly)) { + // This script can only be run via the cli. + print_error('cronerrorclionly', 'admin'); + exit; + } + // This script is being called via the web, so check the password if there is one. + if (!empty($CFG->cronremotepassword)) { + $pass = optional_param('password', '', PARAM_RAW); + if($pass != $CFG->cronremotepassword) { + // wrong password. + print_error('cronerrorpassword', 'admin'); + exit; + } + } + } + + +/// emulate normal session + $SESSION = new object(); + $USER = get_admin(); /// Temporarily, to provide environment for this script + +/// ignore admins timezone, language and locale - use site deafult instead! + $USER->timezone = $CFG->timezone; + $USER->lang = ''; + $USER->theme = ''; + course_setup(SITEID); + +/// send mime type and encoding + if (check_browser_version('MSIE')) { + //ugly IE hack to work around downloading instead of viewing + @header('Content-Type: text/html; charset=utf-8'); + echo ""; //<pre> is not good enough for us here + } else { + //send proper plaintext header + @header('Content-Type: text/plain; charset=utf-8'); + } + +/// no more headers and buffers + while(@ob_end_flush()); + +/// increase memory limit (PHP 5.2 does different calculation, we need more memory now) + @raise_memory_limit('128M'); + +/// Start output log + + $timenow = time(); + + mtrace("Server Time: ".date('r',$timenow)."\n\n"); + + require_once($CFG->dirroot.'/lib/statslib.php'); + + // check we're not before our runtime + $timetocheck = stats_get_base_daily() + $CFG->statsruntimestarthour*60*60 + $CFG->statsruntimestartminute*60; + + if (time() > $timetocheck) { + // process configured number of days as max (defaulting to 31) + $maxdays = empty($CFG->statsruntimedays) ? 31 : abs($CFG->statsruntimedays); + if (stats_cron_daily($maxdays)) { + if (stats_cron_weekly()) { + if (stats_cron_monthly()) { + stats_clean_old(); + } + } + } + @set_time_limit(0); + } else { + mtrace('Next stats run after:'. userdate($timetocheck)); + } + + //Unset session variables and destroy it + @session_unset(); + @session_destroy(); + + mtrace("Statistics Cron script completed correctly"); + + $difftime = microtime_diff($starttime, microtime()); + mtrace("Execution took ".$difftime." seconds"); + +/// finish the IE hack + if (check_browser_version('MSIE')) { + echo ""; + } + +?> diff --git a/course/lib.php b/course/lib.php index 1b1ebea..3aceb07 100644 --- a/course/lib.php +++ b/course/lib.php @@ -187,6 +187,8 @@ function build_mnet_logs_array($hostid, $course, $user=0, $date=0, $order="l.tim return $result; } +require_once($CFG->libdir.'/statslib.php'); + function build_logs_array($course, $user=0, $date=0, $order="l.time ASC", $limitfrom='', $limitnum='', $modname="", $modid=0, $modaction="", $groupid=0) { @@ -246,7 +248,12 @@ function build_logs_array($course, $user=0, $date=0, $order="l.time ASC", $limit if ($date) { $enddate = $date + 86400; - $joins[] = "l.time > '$date' AND l.time < '$enddate'"; + if (stats_get_base_daily($date) == $date) { + // already at midnight, use the optimisation + $joins[] = "l.timeday = '$date'"; + } else { + $joins[] = "l.time > '$date' AND l.time < '$enddate'"; + } } $selector = implode(' AND ', $joins); diff --git a/lib/datalib.php b/lib/datalib.php index af096d4..126e46a 100644 --- a/lib/datalib.php +++ b/lib/datalib.php @@ -1906,6 +1906,16 @@ function add_to_log($courseid, $module, $action, $url='', $info='', $cm=0, $user $url = html_entity_decode($url); // for php < 4.3.0 this is defined in moodlelib.php } + // cribbed from stats_get_base_daily so we needn't include statslib.php + if ($CFG->timezone == 99) { + $timenowday = strtotime(date('d-M-Y', $timenow)); + } else { + $offset = get_timezone_offset($CFG->timezone); + $gtime = $timenow + $offset; + $gtime = intval($gtime / (60*60*24)) * 60*60*24; + $timenowday = $gtime - $offset; + } + // Restrict length of log lines to the space actually available in the // database so that it doesn't cause a DB error. Log a warning so that // developers can avoid doing things which are likely to cause this on a @@ -1931,8 +1941,8 @@ function add_to_log($courseid, $module, $action, $url='', $info='', $cm=0, $user $info = empty($info) ? sql_empty() : $info; // Use proper empties for each database $url = empty($url) ? sql_empty() : $url; - $sql ='INSERT INTO '. $CFG->prefix .'log (time, userid, course, ip, module, cmid, action, url, info) - VALUES (' . "'$timenow', '$userid', '$courseid', '$REMOTE_ADDR', '$module', '$cm', '$action', '$url', '$info')"; + $sql ='INSERT INTO '. $CFG->prefix .'log (time, timeday, userid, course, ip, module, cmid, action, url, info) + VALUES (' . "'$timenow', '$timenowday', '$userid', '$courseid', '$REMOTE_ADDR', '$module', '$cm', '$action', '$url', '$info')"; $result = $db->Execute($sql); diff --git a/lib/db/install.xml b/lib/db/install.xml index fd01eba..6a95a6d 100644 --- a/lib/db/install.xml +++ b/lib/db/install.xml @@ -280,7 +280,8 @@ - + + @@ -290,7 +291,10 @@ - + + + + @@ -1706,4 +1710,4 @@ - \ No newline at end of file + diff --git a/lib/db/upgrade.php b/lib/db/upgrade.php index e3e67d9..f3a0b41 100644 --- a/lib/db/upgrade.php +++ b/lib/db/upgrade.php @@ -3351,6 +3351,58 @@ function xmldb_main_upgrade($oldversion=0) { set_config('filter_mediaplugin_enable_ogg', 1); upgrade_main_savepoint($result, 2007101590.01); } + + if ($result && $oldversion < 2007101591.01) { + // stats processing performance + $table = new XMLDBTable('log'); + $field = new XMLDBField('timeday'); + $field->setAttributes(XMLDB_TYPE_INTEGER, 10, XMLDB_UNSIGNED, XMLDB_NOTNULL, null, null, null, 0); + if (!field_exists($table, $field)) { + $result = $result && add_field($table, $field); + } + + if ($result) { + $index = new XMLDBIndex('userid-course-timeday'); + $index->setFields(array('userid','course','timeday')); + if (!index_exists($table, $index)) { + $result = $result && add_index($table, $index); + } + } + if ($result) { + $index = new XMLDBIndex('course-timeday-action'); + $index->setFields(array('course','timeday','action')); + if (!index_exists($table, $index)) { + $result = $result && add_index($table, $index); + } + } + if ($result) { + $index = new XMLDBIndex('timeday'); + $index->setFields(array('timeday')); + if (!index_exists($table, $index)) { + $result = $result && add_index($table, $index); + } + } + + if ($result) { + $mintimestamp = get_field_select('log', 'MIN(time)', 'timeday = 0'); + $maxtimestamp = get_field_select('log', 'MAX(time)', 'timeday = 0'); + + require_once $CFG->libdir . '/statslib.php'; + + $daystart = stats_get_base_daily($mintimestamp); + do { + $nextdaystart = stats_get_next_day_start($daystart); + + execute_sql("UPDATE {$CFG->prefix}log SET timeday=$daystart WHERE ". + "time >= $daystart AND time < $nextdaystart AND timeday = 0"); + + $daystart = $nextdaystart; + } while ($daystart < $maxtimestamp); + } + + upgrade_main_savepoint($result, 2007101591.01); + } + return $result; } diff --git a/lib/statslib.php b/lib/statslib.php index 6be5da6..588746b 100644 --- a/lib/statslib.php +++ b/lib/statslib.php @@ -153,10 +153,10 @@ function stats_cron_daily($maxdays=1) { $daystart = time(); - $timesql = "l.time >= $timestart AND l.time < $nextmidnight"; - $timesql1 = "l1.time >= $timestart AND l1.time < $nextmidnight"; - $timesql2 = "l2.time >= $timestart AND l2.time < $nextmidnight"; - + $timesql = "l.timeday = $timestart"; + $timesql1 = "l1.timeday = $timestart"; + $timesql2 = "l2.timeday = $timestart"; + stats_daily_progress('init'); diff --git a/version.php b/version.php index be0fdb6..cb10d6d 100644 --- a/version.php +++ b/version.php @@ -6,7 +6,7 @@ // This is compared against the values stored in the database to determine // whether upgrades should be performed (see lib/db/*.php) - $version = 2007101591.00; // YYYYMMDD = date of the 1.9 branch (don't change) + $version = 2007101591.01; // YYYYMMDD = date of the 1.9 branch (don't change) // X = release number 1.9.[0,1,2,3,4,5...] // Y.YY = micro-increments between releases