From 9227685126fda3d54624765a3a3388b7fde2e4ec Mon Sep 17 00:00:00 2001 From: Scott Verbeek Date: Mon, 4 Dec 2023 15:52:58 +1000 Subject: [PATCH] Resend Adhoc task from CLI script (#41) * Add scheduled task to resend failed statements --- classes/task/resend_task.php | 151 +++++++++++++++++++++++++++++++++++ db/tasks.php | 9 +++ lang/en/logstore_xapi.php | 15 ++++ settings.php | 52 +++++++++++- version.php | 2 +- 5 files changed, 226 insertions(+), 3 deletions(-) create mode 100644 classes/task/resend_task.php diff --git a/classes/task/resend_task.php b/classes/task/resend_task.php new file mode 100644 index 000000000..2962de3eb --- /dev/null +++ b/classes/task/resend_task.php @@ -0,0 +1,151 @@ +. + +namespace logstore_xapi\task; + +require_once($CFG->dirroot . '/admin/tool/log/store/xapi/lib.php'); + +/** + * Moves once failed statements back to the queue, ready to resend. + * + * @package logstore_xapi + * @copyright Scott Verbeek + * @license https://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class resend_task extends \core\task\scheduled_task { + + /** + * Get a descriptive name for this task (shown to admins). + * + * @return string + */ + public function get_name() { + return get_string('taskresend', 'logstore_xapi'); + } + + /** + * Do the task, moving once failed statements based upon a configured scope. + */ + public function execute() { + global $DB; + + $settings = new \stdClass(); + $settings->errortype = (string) get_config('logstore_xapi', 'resendtaskerrortype'); + $settings->eventname = (string) get_config('logstore_xapi', 'resendeventname'); + $settings->datefrom = (int) get_config('logstore_xapi', 'resenddatefrom'); + $settings->dateto = (int) get_config('logstore_xapi', 'resenddateto'); + $settings->runtime = (int) get_config('logstore_xapi', 'resendtaskruntime'); + $settings->batch = (int) get_config('logstore_xapi', 'resendbatch'); + + $basetable = XAPI_REPORT_SOURCE_FAILED; + $params = []; + $where = []; + + // Start of sanitization, and applying of scope. + mtrace("Task will stop after {$settings->runtime} seconds have passed, started at " .date('Y-m-d H:i:s T') . "..."); + $starttime = microtime(true); + + mtrace("Task will run in batches of {$settings->batch} records ..."); + + if (!empty($settings->errortype)) { + $settings->errortype = explode(',', $settings->errortype); + $settings->errortype = array_map('intval', $settings->errortype); + list($insql, $inparams) = $DB->get_in_or_equal($settings->errortype, SQL_PARAMS_NAMED, 'errt'); + $where[] = "x.errortype $insql"; + $params = array_merge($params, $inparams); + mtrace('Applied scope for errortype ...'); + } + + if (!empty($settings->eventname)) { + $settings->eventname = explode(',', $settings->eventname); + list($insql, $inparams) = $DB->get_in_or_equal($settings->eventname, SQL_PARAMS_NAMED, 'evt'); + $where[] = "x.eventname $insql"; + $params = array_merge($params, $inparams); + mtrace('Applied scope for eventname ...'); + } + + if (!empty($settings->datefrom) && $settings->datefrom !== false) { + $where[] = 'x.timecreated >= :datefrom'; + $params['datefrom'] = $settings->datefrom; + mtrace('Applied scope for datefrom ...'); + } + + if (!empty($settings->dateto) && $settings->dateto !== false) { + $where[] = 'x.timecreated <= :dateto'; + $params['dateto'] = $settings->dateto; + mtrace('Applied scope for dateto ...'); + } + + if (!empty($settings->datefrom) && !empty($settings->dateto)) { + if ($settings->datefrom > $settings->dateto) { + mtrace(get_string('datetovalidation', 'logstore_xapi')); + exit(2); + } + } + + if (empty($where)) { + $where[] = '1 = 1'; + mtrace('No scope applied, moving all records ...'); + } + + $where = implode(' AND ', $where); + + $sql = "SELECT x.id + FROM {{$basetable}} x + WHERE $where + ORDER BY x.id"; + + $limitfrom = 0; + $limitnum = $settings->batch; + $counttotal = 0; + $countsucc = 0; + $countfail = 0; + + do { + if (microtime(true) - $starttime >= $settings->runtime) { + mtrace("Stopping the task, the maximum runtime has been exceeded ({$settings->runtime} seconds)."); + break; // Exit the loop after the specified runtime + } + + mtrace("Reading at offset {$limitfrom} ...", ' '); + $records = $DB->get_records_sql($sql, $params, $limitfrom, $limitnum); + $count = count($records); + $counttotal += $count; + mtrace("read {$count} records."); + + $eventids = array_keys($records); + + if (empty($eventids)) { + break; + } + + $mover = new \logstore_xapi\log\moveback($eventids, XAPI_REPORT_ID_ERROR); + + if ($mover->execute()) { + $countsucc += $count; + mtrace("$count events successfully sent for reprocessing. Not increasing the offset (records were moved)."); + } else { + $limitfrom += $count; // Increase the offset, when failed to move. + $countfail += $count; + mtrace("$count events failed to send for reprocessing. Increasing the offset by {$count} (records were not moved)."); + } + } while ($count > 0); + + mtrace("Total of {$counttotal} records matched the scope."); + mtrace("Total of {$countsucc} events successfully sent for reprocessing."); + mtrace("Total of {$countfail} events failed to send for reprocessing."); + } +} diff --git a/db/tasks.php b/db/tasks.php index 531963822..95326ca73 100644 --- a/db/tasks.php +++ b/db/tasks.php @@ -54,6 +54,15 @@ 'dayofweek' => '*', 'month' => '*' ), + array( + 'classname' => '\logstore_xapi\task\resend_task', + 'blocking' => 0, + 'minute' => '0', + 'hour' => '18', + 'day' => '*', + 'dayofweek' => '*', + 'month' => '*' + ), array( 'classname' => '\logstore_xapi\task\sendfailednotifications_task', 'blocking' => 0, diff --git a/lang/en/logstore_xapi.php b/lang/en/logstore_xapi.php index 7f89e37c5..bbf55b2c3 100644 --- a/lang/en/logstore_xapi.php +++ b/lang/en/logstore_xapi.php @@ -55,6 +55,7 @@ $string['taskemit'] = 'Emit records to LRS'; $string['taskfailed'] = 'Emit failed records to LRS'; $string['taskhistorical'] = 'Emit historical records to LRS'; +$string['taskresend'] = 'Move failed records to resend'; $string['tasksendfailednotifications'] = 'Send failed notifications'; $string['enablesendingnotifications'] = 'Send notifications?'; $string['enablesendingnotifications_desc'] = 'Control if notifications should be sent to configured recipients.'; @@ -89,6 +90,20 @@ $string['send_response_choices_desc'] = 'Statements for multiple choice and sequencing question answers will be sent to the LRS with the correct response and potential choices'; $string['resendfailedbatches'] = 'Resend failed batches'; $string['resendfailedbatches_desc'] = 'When processing events in batches, try re-sending events in smaller batches if a batch fails. If not selected, the whole batch will not be sent in the event of a failed event.'; +$string['resendtask'] = 'Resend failed logs in background task'; +$string['resendtask_help'] = 'The task resends failed logs in background. It will resend based upon the filter you may pass here.'; +$string['resendtaskruntime'] = 'Runtime'; +$string['resendtaskruntime_help'] = 'The maximum amount of time this task can run in seconds. Note that the default value is also the upper limit.'; +$string['resendtaskerrortype'] = 'Error types'; +$string['resendtaskerrortype_help'] = 'List of (comma seperated) error types (integer) to resend, empty means all.'; +$string['resendeventname'] = 'Event names'; +$string['resendeventname_help'] = 'List of (comma seperated) event names to resend.'; +$string['resenddatefrom'] = 'Date from'; +$string['resenddatefrom_help'] = 'Epoch from date to resend.'; +$string['resenddateto'] = 'Date to'; +$string['resenddateto_help'] = 'Epoch to date to resend.'; +$string['resendbatch'] = 'Batch size'; +$string['resendbatch_help'] = 'The batch size of each move iteration'; $string['type'] = 'Type'; $string['eventname'] = 'Event Name'; $string['username'] = 'Username'; diff --git a/settings.php b/settings.php index d089fd004..d1f82bc5c 100644 --- a/settings.php +++ b/settings.php @@ -69,7 +69,7 @@ $settings->add(new admin_setting_configcheckbox('logstore_xapi/mbox', get_string('mbox', 'logstore_xapi'), get_string('mbox_desc', 'logstore_xapi'), 0)); - + // optional hashing of email address $settings->add(new admin_setting_configcheckbox('logstore_xapi/hashmbox', get_string('hashmbox', 'logstore_xapi'), @@ -86,7 +86,7 @@ $settings->add(new admin_setting_configcheckbox('logstore_xapi/send_username', get_string('send_username', 'logstore_xapi'), get_string('send_username_desc', 'logstore_xapi'), 0)); - + $settings->add(new admin_setting_configcheckbox('logstore_xapi/send_name', get_string('send_name', 'logstore_xapi'), get_string('send_name_desc', 'logstore_xapi'), 1)); @@ -154,6 +154,54 @@ $settings->add(new admin_setting_configmulticheckbox('logstore_xapi/routes', get_string('routes', 'logstore_xapi'), '', $menuroutes, $menuroutes)); + // Resend in background task. + $settings->add(new admin_setting_heading('resendtask', + get_string('resendtask', 'logstore_xapi'), + get_string('resendtask_help', 'logstore_xapi'))); + + $settings->add(new admin_setting_configduration('logstore_xapi/resendtaskruntime', + get_string('resendtaskruntime', 'logstore_xapi'), + get_string('resendtaskruntime_help', 'logstore_xapi'), + get_config('core', 'task_scheduled_max_runtime'), + PARAM_INT + )); + + $settings->add(new admin_setting_configtext('logstore_xapi/resendtaskerrortype', + get_string('resendtaskerrortype', 'logstore_xapi'), + get_string('resendtaskerrortype_help', 'logstore_xapi'), + '', + PARAM_TEXT + )); + + $settings->add(new admin_setting_configtextarea('logstore_xapi/resendeventname', + get_string('resendeventname', 'logstore_xapi'), + get_string('resendeventname_help', 'logstore_xapi'), + '', + PARAM_TEXT + )); + + $settings->add(new admin_setting_configtext('logstore_xapi/resenddatefrom', + get_string('resenddatefrom', 'logstore_xapi'), + get_string('resenddatefrom_help', 'logstore_xapi'), + '', + PARAM_INT + )); + + $settings->add(new admin_setting_configtext('logstore_xapi/resenddateto', + get_string('resenddateto', 'logstore_xapi'), + get_string('resenddateto_help', 'logstore_xapi'), + '', + PARAM_INT + )); + + $settings->add(new admin_setting_configtext('logstore_xapi/resendbatch', + get_string('resendbatch', 'logstore_xapi'), + get_string('resendbatch_help', 'logstore_xapi'), + 12500, + PARAM_INT + )); + + $PAGE->requires->js_call_amd('logstore_xapi/settings', 'init'); // The xAPI Error Log page. diff --git a/version.php b/version.php index 7f101d28a..06c5bf9a2 100644 --- a/version.php +++ b/version.php @@ -27,7 +27,7 @@ defined('MOODLE_INTERNAL') || die(); $plugin->component = 'logstore_xapi'; -$plugin->version = 2022101800; +$plugin->version = 2022101801; $plugin->requires = 2020061500; $plugin->supported = [39, 400]; $plugin->maturity = MATURITY_STABLE;