Index: statslib.php =================================================================== --- statslib.php (revision 921) +++ statslib.php (working copy) @@ -133,6 +133,18 @@ mtrace("Running daily statistics gathering, starting at $timestart:"); + mtrace("...creating stats temporary table: ", false); + if ( ! stats_create_enrollments_view($CFG->statscatdepth, false) ) { + mtrace("ERROR"); + return false; + } + if ( ! stats_create_enrollments_view($CFG->statscatdepth, true) ) { + mtrace("ERROR"); + return false; + } + mtrace("OK"); + + $days = 0; $failed = false; // failed stats flag @@ -154,31 +166,33 @@ $daystart = time(); $timesql = "l.time >= $timestart AND l.time < $nextmidnight"; - $timesql1 = "l1.time >= $timestart AND l1.time < $nextmidnight"; - $timesql2 = "l2.time >= $timestart AND l2.time < $nextmidnight"; + mtrace("...creating logs temporary table: ", false); + if ( ! stats_create_logs_view($timesql) ) { + mtrace("ERROR"); + return false; + } + mtrace("OK"); + stats_daily_progress('init'); - /// find out if any logs available for this day $sql = "SELECT 'x' - FROM {$CFG->prefix}log l - WHERE $timesql"; + FROM {$CFG->prefix}tmp_stats_log l"; $logspresent = get_records_sql($sql, 0, 1); /// process login info first - $sql = "INSERT INTO {$CFG->prefix}stats_user_daily (stattype, timeend, courseid, userid, statsreads) - SELECT 'logins', timeend, courseid, userid, count(statsreads) - FROM ( - SELECT $nextmidnight AS timeend, ".SITEID." AS courseid, l.userid, l.id AS statsreads - FROM {$CFG->prefix}log l - WHERE action = 'login' AND $timesql - ) inline_view - GROUP BY timeend, courseid, userid - HAVING count(statsreads) > 0"; - + $sql = "INSERT INTO {$CFG->prefix}stats_user_daily (stattype, timeend, courseid, userid, statsreads) + SELECT 'logins', " . $nextmidnight . " AS timeend, " . SITEID . " AS courseid, l.userid, count(l.id) + FROM {$CFG->prefix}tmp_stats_log l + WHERE l.action = 'login' AND " . " + l.course = " . SITEID . " + GROUP BY l.userid + HAVING count(l.id) > 0"; + if ($logspresent and !execute_sql($sql, false)) { + mtrace("failed query 1"); $failed = true; break; } @@ -196,12 +210,12 @@ sql_null_from_clause(); if ($logspresent and !execute_sql($sql, false)) { + mtrace("failed query 2"); $failed = true; break; } stats_daily_progress('2'); - // Enrolments and active enrolled users // // Unfortunately, we do not know how many users were registered @@ -217,45 +231,46 @@ $sql = "INSERT INTO {$CFG->prefix}stats_daily (stattype, timeend, courseid, roleid, stat1, stat2) - SELECT 'enrolments', timeend, courseid, roleid, COUNT(DISTINCT userid), 0 - FROM ( - SELECT $nextmidnight AS timeend, pl.courseid, pl.roleid, pl.userid - FROM ( - SELECT DISTINCT ra.roleid, ra.userid, c.id as courseid - FROM {$CFG->prefix}role_assignments ra $enroljoin_na - WHERE $enrolwhere_na - ) pl - ) inline_view - GROUP BY timeend, courseid, roleid"; + SELECT 'enrolments', $nextmidnight AS timeend, courseid, roleid, COUNT(DISTINCT userid), 0 + FROM {$CFG->prefix}tmp_stats_enrollments_na + GROUP BY courseid, roleid"; if (!execute_sql($sql, false)) { + mtrace("failed query 3"); $failed = true; break; } stats_daily_progress('3'); // using table alias in UPDATE does not work in pg < 8.2 - $sql = "UPDATE {$CFG->prefix}stats_daily - SET stat2 = (SELECT COUNT(DISTINCT ra.userid) - FROM {$CFG->prefix}role_assignments ra $enroljoin_na - WHERE ra.roleid = {$CFG->prefix}stats_daily.roleid AND - c.id = {$CFG->prefix}stats_daily.courseid AND - $enrolwhere_na AND - EXISTS (SELECT 'x' - FROM {$CFG->prefix}log l - WHERE l.course = {$CFG->prefix}stats_daily.courseid AND - l.userid = ra.userid AND $timesql)) - WHERE {$CFG->prefix}stats_daily.stattype = 'enrolments' AND + + $sql = "SELECT id, courseid, roleid + FROM {$CFG->prefix}stats_daily + WHERE {$CFG->prefix}stats_daily.stattype = 'enrolments' AND {$CFG->prefix}stats_daily.timeend = $nextmidnight AND {$CFG->prefix}stats_daily.courseid IN (SELECT DISTINCT l.course - FROM {$CFG->prefix}log l - WHERE $timesql)"; + FROM {$CFG->prefix}tmp_stats_log l)"; - if ($logspresent and !execute_sql($sql, false)) { - $failed = true; - break; - } + $res = get_records_sql($sql); + foreach ( $res as $r ) { + + $sql = "UPDATE {$CFG->prefix}stats_daily + SET stat2 = (SELECT COUNT(DISTINCT te.userid) + FROM {$CFG->prefix}tmp_stats_enrollments_na te, {$CFG->prefix}tmp_stats_log l + WHERE l.userid = te.userid AND + l.course = {$r->courseid} AND + te.roleid = {$r->roleid} AND + te.courseid = {$r->courseid}) + WHERE id={$r->id}"; + + if ($logspresent and !execute_sql($sql, false)) { + mtrace("failed query 4"); + $failed = true; + break; + } + } + stats_daily_progress('4'); /// now get course total enrolments (roleid==0) - except frontpage @@ -263,40 +278,65 @@ SELECT 'enrolments', timeend, id, nroleid, COUNT(DISTINCT userid), 0 FROM ( - SELECT $nextmidnight AS timeend, c.id, 0 AS nroleid, ra.userid - FROM {$CFG->prefix}role_assignments ra $enroljoin_na - WHERE c.id <> ".SITEID." AND $enrolwhere_na + SELECT $nextmidnight AS timeend, te.courseid AS id, 0 AS nroleid, te.userid + FROM {$CFG->prefix}tmp_stats_enrollments_na te + WHERE te.courseid <> ".SITEID." ) inline_view GROUP BY timeend, id, nroleid HAVING COUNT(DISTINCT userid) > 0"; if ($logspresent and !execute_sql($sql, false)) { + mtrace("failed query 5"); $failed = true; break; } stats_daily_progress('5'); +/* $sql = "UPDATE {$CFG->prefix}stats_daily - SET stat2 = (SELECT COUNT(DISTINCT ra.userid) - FROM {$CFG->prefix}role_assignments ra $enroljoin_na - WHERE c.id = {$CFG->prefix}stats_daily.courseid AND - $enrolwhere_na AND - EXISTS (SELECT 'x' - FROM {$CFG->prefix}log l - WHERE l.course = {$CFG->prefix}stats_daily.courseid AND - l.userid = ra.userid AND $timesql)) + SET stat2 = (SELECT COUNT(DISTINCT te.userid) + FROM {$CFG->prefix}tmp_stats_enrollments_na te, + {$CFG->prefix}tmp_stats_log l + WHERE te.courseid = {$CFG->prefix}stats_daily.courseid AND + l.course = te.courseid AND + l.userid = te.userid + ) WHERE {$CFG->prefix}stats_daily.stattype = 'enrolments' AND {$CFG->prefix}stats_daily.timeend = $nextmidnight AND {$CFG->prefix}stats_daily.roleid = 0 AND {$CFG->prefix}stats_daily.courseid IN - (SELECT l.course - FROM {$CFG->prefix}log l - WHERE $timesql AND l.course <> ".SITEID.")"; + (SELECT DISTINCT l.course + FROM {$CFG->prefix}tmp_stats_log l + WHERE l.course <> ".SITEID.")"; +*/ - if ($logspresent and !execute_sql($sql, false)) { - $failed = true; - break; - } + $sql = "SELECT * FROM {$CFG->prefix}stats_daily + WHERE stattype = 'enrolments' AND + timeend = $nextmidnight AND + roleid = 0 AND + courseid IN + (SELECT DISTINCT l.course + FROM {$CFG->prefix}tmp_stats_log l + WHERE l.course <> ".SITEID.")"; + + $res = get_records_sql($sql); + foreach ( $res as $r ) { + $sql = "UPDATE {$CFG->prefix}stats_daily + SET stat2 = (SELECT COUNT(DISTINCT te.userid) + FROM {$CFG->prefix}tmp_stats_enrollments_na te, + {$CFG->prefix}tmp_stats_log l + WHERE te.courseid = {$r->courseid} AND + l.course = te.courseid AND + l.userid = te.userid + ) + WHERE id = {$r->id}"; + + if ($logspresent and !execute_sql($sql, false)) { + mtrace("failed query 6"); + $failed = true; + break; + } + } stats_daily_progress('6'); /// frontapge(==site) enrolments total @@ -308,11 +348,12 @@ WHERE u.deleted = 0) AS stat1, (SELECT COUNT(DISTINCT u.id) FROM {$CFG->prefix}user u - JOIN {$CFG->prefix}log l ON l.userid = u.id - WHERE u.deleted = 0 AND $timesql) AS stat2" . + JOIN {$CFG->prefix}tmp_stats_log l ON l.userid = u.id + WHERE u.deleted = 0) AS stat2" . sql_null_from_clause(); if ($logspresent and !execute_sql($sql, false)) { + mtrace("failed query 7"); $failed = true; break; } @@ -332,7 +373,8 @@ WHERE stattype = 'enrolments' AND courseid = ".SITEID." AND roleid = $defaultfproleid AND timeend = $nextmidnight"; if ($logspresent and !execute_sql($sql, false)) { - $failed = true; + mtrace("failed query 8"); + $failed = true; break; } stats_daily_progress('8'); @@ -346,10 +388,11 @@ (SELECT COUNT(DISTINCT u.id) FROM {$CFG->prefix}user u JOIN {$CFG->prefix}log l ON l.userid = u.id - WHERE u.deleted = 0 AND $timesql) AS stat2" . + WHERE u.deleted = 0) AS stat2" . sql_null_from_clause(); if ($logspresent and !execute_sql($sql, false)) { + mtrace("failed query 9"); $failed = true; break; } @@ -361,53 +404,89 @@ } - /// individual user stats (including not-logged-in) in each course, this is slow - reuse this data if possible +/* $sql = "INSERT INTO {$CFG->prefix}stats_user_daily (stattype, timeend, courseid, userid, statsreads, statswrites) SELECT 'activity' AS stattype, $nextmidnight AS timeend, d.courseid, d.userid, (SELECT COUNT('x') - FROM {$CFG->prefix}log l + FROM {$CFG->prefix}tmp_stats_log l WHERE l.userid = d.userid AND - l.course = d.courseid AND $timesql AND + l.course = d.courseid AND l.action IN ($viewactions)) AS statsreads, (SELECT COUNT('x') - FROM {$CFG->prefix}log l + FROM {$CFG->prefix}tmp_stats_log l WHERE l.userid = d.userid AND - l.course = d.courseid AND $timesql AND + l.course = d.courseid AND l.action IN ($postactions)) AS statswrites FROM (SELECT DISTINCT u.id AS userid, l.course AS courseid - FROM {$CFG->prefix}user u, {$CFG->prefix}log l - WHERE u.id = l.userid AND $timesql + FROM {$CFG->prefix}user u, {$CFG->prefix}tmp_stats_log l + WHERE u.id = l.userid UNION SELECT 0 AS userid, ".SITEID." AS courseid" . sql_null_from_clause() . ") d"; - // can not use group by here because pg can not handle it :-( +*/ - if ($logspresent and !execute_sql($sql, false)) { - $failed = true; - break; - } + $sql = "SELECT DISTINCT u.id AS userid, l.course AS courseid + FROM {$CFG->prefix}user u, {$CFG->prefix}tmp_stats_log l + WHERE u.id = l.userid + UNION + SELECT 0 AS userid, ".SITEID." AS courseid" . sql_null_from_clause(); + + $res = get_records_sql($sql); + + $a_values = array(); + + foreach ( $res as $r ) { + + $statsreads = count_records_select('tmp_stats_log', "userid={$r->userid} AND course={$r->courseid} AND action IN ($viewactions)"); + $statswrites = count_records_select('tmp_stats_log', "userid={$r->userid} AND course={$r->courseid} AND action IN ($postactions)"); + + $a_values[] = "('activity',$nextmidnight, {$r->courseid}, {$r->userid}, $statsreads, $statswrites)"; + + if ( (count($a_values) % 100) == 0 ) { + $sql = "INSERT INTO {$CFG->prefix}stats_user_daily (stattype, timeend, courseid, userid, statsreads, statswrites) VALUES "; + $sql .= implode(",", $a_values); + if ( $logspresent && ! execute_sql($sql, false) ) { + mtrace("failed query 10"); + $failed = true; + break; + } + $a_values = array(); + } + } + if ( count($a_values) > 0 ) { + $sql = "INSERT INTO {$CFG->prefix}stats_user_daily (stattype, timeend, courseid, userid, statsreads, statswrites) VALUES "; + $sql .= implode(",", $a_values); + if ( $logspresent && ! execute_sql($sql, false) ) { + mtrace("failed query 10"); + $failed = true; + break; + } + $a_values = null; + } + stats_daily_progress('10'); - /// how many view/post actions in each course total $sql = "INSERT INTO {$CFG->prefix}stats_daily (stattype, timeend, courseid, roleid, stat1, stat2) SELECT 'activity' AS stattype, $nextmidnight AS timeend, c.id AS courseid, 0, (SELECT COUNT('x') - FROM {$CFG->prefix}log l1 - WHERE l1.course = c.id AND l1.action IN ($viewactions) AND - $timesql1) AS stat1, + FROM {$CFG->prefix}tmp_stats_log l1 + WHERE l1.course = c.id AND l1.action IN ($viewactions) + ) AS stat1, (SELECT COUNT('x') - FROM {$CFG->prefix}log l2 - WHERE l2.course = c.id AND l2.action IN ($postactions) AND - $timesql2) AS stat2 + FROM {$CFG->prefix}tmp_stats_log l2 + WHERE l2.course = c.id AND l2.action IN ($postactions) + ) AS stat2 FROM {$CFG->prefix}course c - WHERE EXISTS (SELECT 'x' - FROM {$CFG->prefix}log l - WHERE l.course = c.id and $timesql)"; + WHERE EXISTS (SELECT 'x' + FROM {$CFG->prefix}tmp_stats_log l + WHERE l.course = c.id )"; + stats_daily_progress('11'); if ($logspresent and !execute_sql($sql, false)) { + mtrace("failed query 11"); $failed = true; break; } @@ -420,24 +499,24 @@ SELECT 'activity', timeend, courseid, roleid, SUM(statsreads), SUM(statswrites) FROM ( - SELECT $nextmidnight AS timeend, pl.courseid, pl.roleid, sud.statsreads, sud.statswrites + SELECT $nextmidnight AS timeend, te.courseid, te.roleid, sud.statsreads, sud.statswrites FROM {$CFG->prefix}stats_user_daily sud, - (SELECT DISTINCT ra.userid, ra.roleid, c.id AS courseid - FROM {$CFG->prefix}role_assignments ra $enroljoin - WHERE c.id <> ".SITEID." AND - ra.roleid <> $guestrole->id AND - ra.userid <> $guest->id AND - $enrolwhere - ) pl - WHERE sud.userid = pl.userid AND - sud.courseid = pl.courseid AND + {$CFG->prefix}tmp_stats_enrollments te + WHERE te.courseid <> " . SITEID . " AND + te.roleid <> $guestrole->id AND + te.userid <> $guest->id AND + sud.userid = te.userid AND + sud.courseid = te.courseid AND sud.timeend = $nextmidnight AND sud.stattype='activity' ) inline_view GROUP BY timeend, courseid, roleid HAVING SUM(statsreads) > 0 OR SUM(statswrites) > 0"; + + if ($logspresent and !execute_sql($sql, false)) { + mtrace("failed query 12"); $failed = true; break; } @@ -456,15 +535,16 @@ WHERE sud.timeend = $nextmidnight AND sud.courseid <> ".SITEID." AND sud.stattype='activity' AND (sud.userid = $guest->id OR sud.userid - NOT IN (SELECT ra.userid - FROM {$CFG->prefix}role_assignments ra $enroljoin - WHERE c.id <> ".SITEID." AND ra.roleid <> $guestrole->id AND - $enrolwhere)) + NOT IN (SELECT DISTINCT te.userid + FROM {$CFG->prefix}tmp_stats_enrollments te + WHERE te.courseid <> ".SITEID." AND te.roleid <> $guestrole->id + )) ) inline_view GROUP BY timeend, courseid, nroleid HAVING SUM(statsreads) > 0 OR SUM(statswrites) > 0"; if ($logspresent and !execute_sql($sql, false)) { + mtrace("failed query 13"); $failed = true; break; } @@ -478,13 +558,12 @@ FROM ( SELECT $nextmidnight AS timeend, pl.courseid, pl.roleid, sud.statsreads, sud.statswrites FROM {$CFG->prefix}stats_user_daily sud, - (SELECT DISTINCT ra.userid, ra.roleid, c.id AS courseid - FROM {$CFG->prefix}role_assignments ra $enroljoin - WHERE c.id = ".SITEID." AND - ra.roleid <> $defaultfproleid AND - ra.roleid <> $guestrole->id AND - ra.userid <> $guest->id AND - $enrolwhere + (SELECT DISTINCT te.userid, te.roleid, te.courseid + FROM {$CFG->prefix}tmp_stats_enrollments te + WHERE te.courseid = ".SITEID." AND + te.roleid <> $defaultfproleid AND + te.roleid <> $guestrole->id AND + te.userid <> $guest->id ) pl WHERE sud.userid = pl.userid AND sud.courseid = pl.courseid AND @@ -494,7 +573,9 @@ GROUP BY timeend, courseid, roleid HAVING SUM(statsreads) > 0 OR SUM(statswrites) > 0"; + if ($logspresent and !execute_sql($sql, false)) { + mtrace("failed query 14"); $failed = true; break; } @@ -511,15 +592,16 @@ WHERE sud.timeend = $nextmidnight AND sud.courseid = ".SITEID." AND sud.stattype='activity' AND sud.userid <> $guest->id AND sud.userid <> 0 AND sud.userid - NOT IN (SELECT ra.userid - FROM {$CFG->prefix}role_assignments ra $fpjoin - WHERE c.id = ".SITEID." AND ra.roleid <> $guestrole->id AND - ra.roleid <> $defaultfproleid AND $fpwhere) + NOT IN (SELECT te.userid + FROM {$CFG->prefix}tmp_stats_enrollments_na te + WHERE te.courseid = ".SITEID." AND te.roleid <> $guestrole->id AND + te.roleid <> $defaultfproleid ) ) inline_view GROUP BY timeend, courseid, nroleid HAVING SUM(statsreads) > 0 OR SUM(statswrites) > 0"; if ($logspresent and !execute_sql($sql, false)) { + mtrace("failed query 15"); $failed = true; break; } @@ -543,6 +625,7 @@ HAVING SUM(statsreads) > 0 OR SUM(statswrites) > 0"; if ($logspresent and !execute_sql($sql, false)) { + mtrace("failed query 16"); $failed = true; break; } @@ -830,7 +913,158 @@ return true; } + + +require_once($CFG->libdir . '/ddllib.php'); + /** + * Creates tmp log table + * @param timesql string time log condition + * @returns boolen success (true) or failure(false) + */ +function stats_create_logs_view ( $timesql ) { + + global $CFG; + + if ( ! execute_sql("DROP TABLE IF EXISTS {$CFG->prefix}tmp_stats_log", false) ) { + mtrace("... error droping tmp_stats_log"); + return false; + } + + if (!execute_sql("CREATE TABLE {$CFG->prefix}tmp_stats_log AS + SELECT * + FROM {$CFG->prefix}log l + WHERE ".$timesql, false)) { + mtrace("...error creating daily SQL temp table, {$CFG->prefix}tmp_stats_log"); + return false; + } + + if (!execute_sql("CREATE INDEX {$CFG->prefix}tmp_stats_log_time_idx ON {$CFG->prefix}tmp_stats_log ( time )", false)) { + mtrace("...error creating daily SQL index, {$CFG->prefix}tmp_stats_log_time_idx"); + return false; + } + + if (!execute_sql("CREATE INDEX {$CFG->prefix}tmp_stats_log_course_idx ON {$CFG->prefix}tmp_stats_log ( course )", false)) { + mtrace("...error creating daily SQL index, {$CFG->prefix}tmp_stats_log_course_idx"); + return false; + } + + if (!execute_sql("CREATE INDEX {$CFG->prefix}tmp_stats_log_action_idx ON {$CFG->prefix}tmp_stats_log ( action )", false)) { + mtrace("...error creating daily SQL index, {$CFG->prefix}tmp_stats_log_action_idx"); + return false; + } + + return true; +} + +/** + * Creates temporary tables for enrollments + * @param int $limit number of max parent course categories + * @param bool $includedoanything include also admins + * @return array ra join and where string + */ + +function stats_create_enrollments_view ( $limit, $includedoanything ) { + + global $CFG; + + $caps = $includedoanything ? "(rc.capability = 'moodle/course:view' OR rc.capability = 'moodle/site:doanything')" : "rc.capability = 'moodle/course:view'"; + $table_name = $includedoanything ? "{$CFG->prefix}tmp_stats_enrollments" : "{$CFG->prefix}tmp_stats_enrollments_na"; + + $sql = "DROP TABLE IF EXISTS $table_name"; + if ( ! execute_sql($sql, false) ) { + return false; + } + + $a_sql = array ( "CREATE TABLE $table_name ( + roleid int(10) unsigned, + userid int(10) unsigned, + courseid int(10) unsigned, + PRIMARY KEY (roleid, userid, courseid) + )", + "create index {$CFG->prefix}tmp_stats_enrollments_courseid_idx on $table_name ( courseid )", + "create index {$CFG->prefix}tmp_stats_enrollments_userid_idx on $table_name ( userid )", + "create index {$CFG->prefix}tmp_stats_enrollments_roleid_idx on $table_name ( roleid)"); + + foreach ( $a_sql as $sql ) { + if ( ! execute_sql($sql, false) ) { + return false; + } + } + + $sql = "INSERT INTO $table_name + SELECT ra.roleid, ra.userid, c.id as courseid + FROM {$CFG->prefix}role_assignments ra, + {$CFG->prefix}course c, + {$CFG->prefix}role_capabilities rc, + {$CFG->prefix}context ctx + WHERE ctx.id = ra.contextid AND + ra.roleid = rc.roleid AND + c.id = ctx.instanceid AND + $caps AND + rc.permission = 1 AND + rc.contextid = " . SYSCONTEXTID . " AND + ctx.contextlevel = " . CONTEXT_COURSE . " + UNION + SELECT ra.roleid, ra.userid, c.id as courseid + FROM {$CFG->prefix}role_assignments ra, + {$CFG->prefix}course c, + {$CFG->prefix}role_capabilities rc, + {$CFG->prefix}context ctx + WHERE ctx.id = ra.contextid AND + ra.roleid = rc.roleid AND + $caps AND + rc.permission = 1 AND + rc.contextid = " . SYSCONTEXTID . " AND + ctx.contextlevel = " . CONTEXT_SYSTEM; + + /** Union for role assignments at course category level + */ + + if ( $limit >= 1 ) { + $join = ""; + $where =" AND ("; + for($i=1; $i<=$limit; $i++) { + if ($i == 1) { + $join .= " LEFT OUTER JOIN {$CFG->prefix}course_categories cc1 + ON cc1.id = c.category"; + $where .= " (cc1.id = ctx.instanceid )"; + } else { + $j = $i-1; + $join .= " LEFT OUTER JOIN {$CFG->prefix}course_categories cc$i + ON cc$i.id = cc$j.parent"; + $where .= " OR (cc$i.id = ctx.instanceid )"; + } + } + + $where = $where . ")"; + + $sql = $sql . "\nUNION + SELECT DISTINCT ra.roleid, ra.userid, c.id as courseid + FROM mdl_role_assignments ra + JOIN + mdl_context ctx ON ctx.id = ra.contextid + CROSS JOIN + mdl_course c + JOIN mdl_role_capabilities rc ON rc.roleid = ra.roleid + $join + WHERE + (rc.capability = 'moodle/course:view' ) AND + rc.permission = 1 AND + rc.contextid = 2 AND + ctx.contextlevel = " . CONTEXT_COURSECAT . + $where; + } + + if ( ! execute_sql($sql, false) ) + return false; + + return true; + + +} + +/** * Returns simplified enrolment sql join data * @param int $limit number of max parent course categories * @param bool $includedoanything include also admins