diff --git a/README.md b/README.md index 18e40966..88f138ad 100644 --- a/README.md +++ b/README.md @@ -108,6 +108,10 @@ It also requires PHP's `zip` extension installed and enabled. 5. `./deploy-service ARCVService_v(-[beta|RC]X).tgz service_v(-[beta|RC]X)` 6. update the `.env` file +# MVL Export + +See [MVL-EXPORT.md](./docs/MVL-EXPORT.md). + # Copyright This project was developed by : diff --git a/app/Bundle.php b/app/Bundle.php index 87adc422..99141447 100644 --- a/app/Bundle.php +++ b/app/Bundle.php @@ -19,7 +19,7 @@ * @property Carer $collectingCarer * @property Centre $disbursingCentre * @property User $disbursingUser - * @property string $disbursed_at + * @property Carbon $disbursed_at */ class Bundle extends Model { diff --git a/app/Console/Commands/CreateMasterVoucherLogReport.php b/app/Console/Commands/CreateMasterVoucherLogReport.php index 5812b668..5ad847ba 100644 --- a/app/Console/Commands/CreateMasterVoucherLogReport.php +++ b/app/Console/Commands/CreateMasterVoucherLogReport.php @@ -9,6 +9,7 @@ use Illuminate\Console\Command; use Illuminate\Support\Carbon; use Illuminate\Support\Facades\Storage; +use Log; use PDO; use ZipStream\Exception\OverflowException; use ZipStream\Option\Archive; @@ -16,7 +17,6 @@ class CreateMasterVoucherLogReport extends Command { - const ROW_LIMIT = 900000; /** * The name and signature of the console command. * @@ -26,28 +26,29 @@ class CreateMasterVoucherLogReport extends Command {--force : Execute without confirmation, eg for automation} {--no-zip : Don\'t wrap files in a single archive} {--plain : Don\'t encrypt contents of files} - {--from= : start date, will default to the start of this financial year} - {--to= : end date, will default to "now"} - {--chunk-size= : how many records to process each chunk} '; + /** * The console command description. * * @var string $description */ protected $description = 'Creates, encrypts and stores the MVL report under in /storage'; + /** * The default disk we want. * * @var string $disk */ private $disk; + /** * The default archive name. * * @var string $archiveName */ private $archiveName; + /** * The sheet headers. * @@ -74,29 +75,23 @@ class CreateMasterVoucherLogReport extends Command 'Void Reason', 'Date file was Downloaded', ]; + /** * The date that we care about for last year's data. - * @var string $endDate + * @var string $cutOffDate */ - private string $endDate; - /** - * The date to start collecting vouchers from - * @var string $startDate - */ - private string $startDate; - /** - * How many records to process per tick - * - * @var int $chunkSize - */ - private int $chunkSize = 5000; + private string $cutOffDate; + /** - * @var ZipStream $za ; + * @var ZipStream $za; */ private ZipStream $za; - // Excel can't deal with large CSVs private $zaOutput; + + // Excel can't deal with large CSVs + public const ROW_LIMIT = 900000; + /** * The report's query template * @@ -206,6 +201,43 @@ public function __construct() $this->archiveName = config('arc.mvl_filename'); } + /** + * @return void + */ + public function initSettings(): void + { + $thisYearsApril = Carbon::parse('april')->startOfMonth(); + // $years = ($thisYearsApril->isPast()) ? 2 : 1; + // $this->cutOffDate = $thisYearsApril->subYearsNoOverflow($years)->format('Y-m-d'); + if ($thisYearsApril->isFuture()) { + $this->cutOffDate = $thisYearsApril->subYearsNoOverflow(1)->format('Y-m-d'); + } else { + $this->cutOffDate = $thisYearsApril->format('Y-m-d'); + } + + + // Set the disk + $this->disk = ($this->option('plain')) + ? 'local' + : 'enc'; + + if (!$this->option('no-zip')) { + // Open the file for writing at the correct location + $path = Storage::path($this->disk) . '/' . $this->archiveName; + + // Encrypt the output stream if the user hasn't asked for it to be plain. + if (!$this->option('plain')) { + $path = 'ssw://' . $path; + } + + // Stream directly to what is either a file or a file wrapped in a secret stream. + $options = new Archive(); + $this->zaOutput = fopen($path, 'w'); + $options->setOutputStream($this->zaOutput); + $this->za = new ZipStream(null, $options); + } + } + /** * Execute the console command. * @@ -222,21 +254,16 @@ public function handle() $rows = []; $continue = true; $lookups = 0; + $chunkSize = 50000; $mem = memory_get_usage(); - $this->info(sprintf( - "Starting MVL export from %s to %s in chunks of %d. Starting mem=%s", - $this->startDate, - $this->endDate, - $this->chunkSize, - TextFormatter::formatBytes($mem) - )); + Log::info("starting query, mem :" . $mem); while ($continue) { - $chunk = $this->execQuery($this->chunkSize, $this->chunkSize * $lookups); + $chunk = $this->execQuery($chunkSize, $chunkSize * $lookups); // when we're at the tail, quit next round - if (count($chunk) < $this->chunkSize) { + if (count($chunk) < $chunkSize) { $continue = false; } @@ -259,12 +286,12 @@ public function handle() unset($chunk); $mem = memory_get_usage(); - $this->info(sprintf("Chunk %d, Mem: %s", $lookups, TextFormatter::formatBytes($mem))); + Log::info($lookups . 'x' . $chunkSize . ', skip ' . $chunkSize * $lookups . ",mem :" . $mem); } - $this->info("Finished query, meme:" . TextFormatter::formatBytes($mem)); - $this->info("Using " . $this->disk); - $this->info("beginning file write, mem:" . TextFormatter::formatBytes(memory_get_usage())); + Log::info("finished query, meme:" . $mem); + Log::info("using " . $this->disk); + Log::info("beginning file write, mem:" . memory_get_usage()); $this->writeMultiPartMVL($rows); @@ -276,8 +303,8 @@ public function handle() try { $this->za->finish(); } catch (OverflowException $e) { - $this->error($e->getMessage()); - $this->error("Overflow when attempting to finish a significantly large Zip file"); + Log::error($e->getMessage()); + Log::error("Overflow when attempting to finish a significantly large Zip file"); exit(1); } @@ -290,79 +317,6 @@ public function handle() exit(0); } - /** - * @return void - */ - public function initSettings(): void - { - $from = $this->option('from'); - if ($from) { - $carbonDate = Carbon::parse($from); - if ($carbonDate) { - $this->startDate = $carbonDate->toDateString(); - } else { - // Not a date - $this->error("From date was not a valid date: " . $from); - } - } else { - $this->startDate = $this->getStartOfThisFinancialYear()->toDateString(); - } - - $to = $this->option('to'); - if ($to) { - $carbonDate = Carbon::parse($to); - if ($carbonDate) { - $this->endDate = $carbonDate->toDateString(); - } else { - // Not a date - $this->error("To date was not a valid date: " . $to); - } - } else { - $this->endDate = Carbon::now()->toDateString(); - } - - $chunkSize = $this->option("chunk-size"); - if ($chunkSize) { - $this->chunkSize = intval($chunkSize); - if ($this->chunkSize === 0) { - $this->error("Chunk size does not seem to be a valid int: " . $chunkSize); - } - } - - // Set the disk - $this->disk = ($this->option('plain')) - ? 'local' - : 'enc'; - - if (!$this->option('no-zip')) { - // Open the file for writing at the correct location - $path = Storage::path($this->disk) . '/' . $this->archiveName; - - // Encrypt the output stream if the user hasn't asked for it to be plain. - if (!$this->option('plain')) { - $path = 'ssw://' . $path; - } - - // Stream directly to what is either a file or a file wrapped in a secret stream. - $options = new Archive(); - $this->zaOutput = fopen($path, 'w'); - $options->setOutputStream($this->zaOutput); - $this->za = new ZipStream(null, $options); - } - } - - private function getStartOfThisFinancialYear(): bool|Carbon - { - $currentDate = Carbon::now(); - $financialYearStartDate = Carbon::create($currentDate->year, 4, 1, 0, 0, 0); - - if ($currentDate->lt($financialYearStartDate)) { - // If the current date is before April 1st, subtract one year - $financialYearStartDate->subYear(); - } - return $financialYearStartDate; - } - /** * Warn the user before they execute. * @@ -374,6 +328,7 @@ public function warnUser(): bool return $this->confirm('Do you wish to continue?'); } + /** * returns a query chunk * @param int $limit @@ -390,26 +345,34 @@ private function execQuery(int $limit, int $offset): bool|array } /** - * This should be part of the query - * @param $voucher + * Encrypts and stashes files. + * + * @param String $name + * @param String $csv * @return bool */ - public function rejectThisVoucher($voucher): bool + public function writeOutput(string $name, string $csv): bool { - if (is_null($voucher['Reimbursed Date'])) { - // exit early - return false; - } - $reimbursedTime = strtotime(DateTime::createFromFormat('d/m/Y', $voucher['Reimbursed Date'])->format('Y-m-d')); + try { + $filename = sprintf("%s.csv", preg_replace('/\s+/', '_', $name)); - // return true, if any of these are true - return - // are all the fields we care about null? - $this->containsOnlyNull($voucher) || - // is this date filled? - !is_null($voucher['Void Voucher Date']) || - // is this date dilled *and* less than the cut-off date - ($reimbursedTime >= strtotime($this->startDate) && $reimbursedTime < strtotime($this->endDate)); + if (isset($this->za)) { + // Encryption, if enabled, is handled at the creation of our ZipStream. The stream is directed through + // an encrypted wrapper before it writes to the disk. + $this->za->addFile($filename, $csv); + } elseif ($this->option('plain')) { + Storage::disk($this->disk)->put($filename, $csv); + } else { + // TODO : Consider redesigning the CLI such that this is not even possible to express. + Log::error('Encrypted output is not supported with --no-zip, consider adding --plain if you\'re SURE you want to write this data to the disk unencrypted.'); + } + } catch (Exception $e) { + // Could be Storage or LaravelExcelWriterException related + Log::error($e->getMessage()); + Log::error(class_basename($this) . ": Failed to write file for '" . $csv->getTitle() . "'"); + exit(1); + } + return true; } /** @@ -442,6 +405,31 @@ public function containsOnlyNull(array $array): bool return true; } + /** + * This should be part of the query + * @param $voucher + * @return bool + */ + public function rejectThisVoucher($voucher): bool + { + // return true, if any of these are true + return + // are all the fields we care about null? + $this->containsOnlyNull($voucher) || + // is this date filled? + !is_null($voucher['Void Voucher Date']) || + // is this date dilled *and* less than the cut-off date + ( + !is_null($voucher['Reimbursed Date']) && + strtotime( + DateTime::createFromFormat( + 'd/m/Y', + $voucher['Reimbursed Date'] + )->format('Y-m-d') + ) < strtotime($this->cutOffDate) + ); + } + /** * @param $rows * @return void @@ -466,7 +454,7 @@ public function writeMultiPartMVL($rows): void fputcsv($fileHandleAll, $row); // calculate the file number - $calcFileNum = 1 + intdiv($index, self::ROW_LIMIT); + $calcFileNum = 1 + intdiv($index, self::ROW_LIMIT); // has it increased? $nextFile = ($calcFileNum > $fileNum); @@ -474,7 +462,7 @@ public function writeMultiPartMVL($rows): void if ($nextFile || $last_key === $index) { // stash and close this file rewind($fileHandleAll); - $this->writeOutput('PART' . $fileNum, stream_get_contents($fileHandleAll)); + $this->writeOutput('PART'. $fileNum, stream_get_contents($fileHandleAll)); fclose($fileHandleAll); // update the fileNum $fileNum = $calcFileNum; @@ -482,37 +470,6 @@ public function writeMultiPartMVL($rows): void } } - /** - * Encrypts and stashes files. - * - * @param String $name - * @param String $csv - * @return bool - */ - public function writeOutput(string $name, string $csv): bool - { - try { - $filename = sprintf("%s.csv", preg_replace('/\s+/', '_', $name)); - - if (isset($this->za)) { - // Encryption, if enabled, is handled at the creation of our ZipStream. The stream is directed through - // an encrypted wrapper before it writes to the disk. - $this->za->addFile($filename, $csv); - } elseif ($this->option('plain')) { - Storage::disk($this->disk)->put($filename, $csv); - } else { - // TODO : Consider redesigning the CLI such that this is not even possible to express. - $this->error('Encrypted output is not supported with --no-zip, consider adding --plain if you\'re SURE you want to write this data to the disk unencrypted.'); - } - } catch (Exception $e) { - // Could be Storage or LaravelExcelWriterException related - $this->error($e->getMessage()); - $this->error(class_basename($this) . ": Failed to write file for '" . $csv . "'"); - exit(1); - } - return true; - } - /** * @param $rows * @return void diff --git a/app/Console/Commands/MvlExport.php b/app/Console/Commands/MvlExport.php new file mode 100644 index 00000000..540a18bf --- /dev/null +++ b/app/Console/Commands/MvlExport.php @@ -0,0 +1,172 @@ +initSettings(); + + $this->info( + sprintf( + "Starting voucher export from %s to %s in chunks of %d.", + $this->startDate->format("Y/m/d"), + $this->endDate->format("Y/m/d"), + $this->chunkSize, + ) + ); + + $query = Voucher::where("currentstate", "=", "reimbursed") + ->whereBetween('updated_at', [$this->startDate, $this->endDate]) + ->orderBy("updated_at"); + + $this->info("Counting rows"); + $this->time = microtime(true); + $count = $query->count(); + $this->info(sprintf( + "Got %d vouchers in %s", + $count, + TextFormatter::secondsToTime(microtime(true) - $this->time) + )); + + $today = Carbon::now()->format("Y-m-d"); + + $outputDir = sprintf("%s/mvl/export/$today", Storage::path(self::DISK)); + if (!is_dir($outputDir)) { + mkdir($outputDir, 0755, true); + } + + $this->info(sprintf( + "Starting memory: %s", + TextFormatter::formatBytes(memory_get_usage()) + )); + $offset = 0; + while ($offset < $count) { + $this->time = microtime(true); + $filename = sprintf( + "%s/vouchers.%s-to-%s.%04d.arcx", + $outputDir, + $this->startDate->format("Ymd"), + $this->endDate->format("Ymd"), + floor(($offset + 1) / $this->chunkSize) + ); + + $voucherIds = $query->offset($offset) + ->limit($this->chunkSize) + ->pluck('id') + ->toArray(); + + file_put_contents($filename, join("\n", $voucherIds)); + $this->info(sprintf( + "Wrote %d voucher ids to %s in %s, Mem: %s", + count($voucherIds), + $filename, + TextFormatter::secondsToTime(microtime(true) - $this->time), + TextFormatter::formatBytes(memory_get_usage()) + )); + + $offset += $this->chunkSize; + } + } + + /** + * @return void + */ + public function initSettings(): void + { + $from = $this->option('from'); + if ($from) { + try { + $this->startDate = Carbon::createFromFormat('d/m/Y', $from, 'UTC'); + } catch (InvalidFormatException $exception) { + // Not a date + $this->error("From date was not a valid date: " . $from); + exit(1); + } + } else { + // We were going to start this financial year + // $this->startDate = $this->getStartOfThisFinancialYear()->format('d/m/Y'); + $this->startDate = Carbon::createFromFormat('d/m/Y', "01/01/1970", 'UTC'); + } + + $to = $this->option('to'); + if ($to) { + try { + $this->endDate = Carbon::createFromFormat('d/m/Y', $to, 'UTC'); + } catch (InvalidFormatException $exception) { + // Not a date + $this->error("To date was not a valid date: " . $to); + exit(1); + } + } else { + // We were going to use till now + // $this->endDate = Carbon::now()->format('d/m/Y'); + $this->endDate = Carbon::createFromFormat('d/m/Y', "31/08/2023", 'UTC'); + } + + $chunkSize = $this->option("chunk-size"); + if ($chunkSize) { + $this->chunkSize = intval($chunkSize); + if ($this->chunkSize === 0) { + $this->error("Chunk size does not seem to be a valid int: " . $chunkSize); + } + } + + } +} \ No newline at end of file diff --git a/app/Console/Commands/MvlProcess.php b/app/Console/Commands/MvlProcess.php new file mode 100644 index 00000000..3d4e670d --- /dev/null +++ b/app/Console/Commands/MvlProcess.php @@ -0,0 +1,105 @@ +argument("file"); + $this->info(sprintf("Reading ids from %s", $in_file)); + if (!file_exists($in_file)) { + $this->error(sprintf("File not found: %s", $in_file)); + } + + $targetDir = dirname($in_file); + if (!file_exists($targetDir . "/_headers.csv")) { + if (!is_dir($targetDir)) { + mkdir($targetDir, 0755, true); + } + $fh = fopen($targetDir . "/_headers.csv", "w"); + fputcsv($fh, $this->headers); + fclose($fh); + } + + $out_file = $targetDir . "/" . basename($in_file, ".txt") . ".csv"; + $this->info(sprintf("Writing ids to %s", $out_file)); + + $fh_out = fopen($out_file, 'w'); + $count = 0; + $startTime = microtime(true); + $time = microtime(true); + $lines = explode("\n", file_get_contents($in_file)); + + $sharedData = [ null, null, $now = Carbon::now()->format("Y/m/d")]; + foreach ($lines as $id) { + $v = Voucher::find($id); + if ($v) { + fputcsv($fh_out, array_merge($v->deepExport(), $sharedData)); + if ($count++ % self::TICK_SIZE === 0) { + $this->info(sprintf( + "Writing vouchers %d to %d, Mem: %s, elapsed time %f seconds", + $count, + $count + self::TICK_SIZE - 1, + TextFormatter::formatBytes(memory_get_usage()), + (microtime(true) - $time), + )); + $time = microtime(true); + } + } + } + + $this->info("Total time: " . TextFormatter::secondsToTime(ceil(microtime(true) - $startTime))); + fclose($fh_out); + } +} diff --git a/app/Delivery.php b/app/Delivery.php index 1291ce87..1c6272f6 100644 --- a/app/Delivery.php +++ b/app/Delivery.php @@ -6,9 +6,10 @@ use Illuminate\Database\Eloquent\Model; use Illuminate\Database\Eloquent\Relations\BelongsTo; use Illuminate\Database\Eloquent\Relations\HasMany; +use Illuminate\Support\Carbon; /** - * @property string $dispatched_at + * @property Carbon $dispatched_at * @property Centre $centre * @property Voucher[] $vouchers */ diff --git a/app/Family.php b/app/Family.php index 58cb4cbd..e9f29936 100644 --- a/app/Family.php +++ b/app/Family.php @@ -26,6 +26,7 @@ * @property Note[] $notes * @property Registration[] $registrations * @property Centre $initialCentre + * @property string $rvid * */ class Family extends Model implements IEvaluee diff --git a/app/Services/TextFormatter.php b/app/Services/TextFormatter.php index abebfb46..9f766fff 100644 --- a/app/Services/TextFormatter.php +++ b/app/Services/TextFormatter.php @@ -38,4 +38,27 @@ public static function formatBytes(int $bytes, int $precision = 2): string // Format the number with the specified precision return round($bytes, $precision) . ' ' . $units[$pow]; } -} \ No newline at end of file + + /** + * Produce a human-readable time elapsed sting from a number of seconds. + * + * @param $seconds + * @return string + */ + public static function secondsToTime($seconds): string + { + $secs = (int)$seconds; + $format = '%s seconds'; + if ($secs > 60 * 60 * 24) { + $format = '%a days, %h hours, %i minutes and %s seconds'; + } elseif ($secs > 60 * 60) { + $format = '%h hours, %i minutes and %s seconds'; + } elseif ($secs > 60) { + $format = '%i minutes and %s seconds'; + } // else Use default value set above + + $dtF = new \DateTime('@0'); + $dtT = new \DateTime("@$secs"); + return $dtF->diff($dtT)->format($format); + } +} diff --git a/app/Voucher.php b/app/Voucher.php index 2b396d6c..aac0baaa 100644 --- a/app/Voucher.php +++ b/app/Voucher.php @@ -4,8 +4,8 @@ use App\Traits\Statable; use Auth; -use DB; use DateTimeInterface; +use DB; use Eloquent; use Illuminate\Database\Eloquent\Builder; use Illuminate\Database\Eloquent\Model; @@ -18,19 +18,26 @@ /** * @mixin Eloquent + * @property integer id * @property string $code * @property Sponsor $sponsor * @property Trader $trader * @property Bundle $bundle * @property Delivery $delivery + * @property string $rvid * @property VoucherState $paymentPendedOn * @property VoucherState $recordedOn * @property VoucherState $reimbursedOn */ class Voucher extends Model { - use Statable; // import the state transition stuff. - use SoftDeletes; // import soft delete. + use Statable; + + // import the state transition stuff. + + use SoftDeletes; + + // import soft delete. protected $dates = ['deleted_at']; const HISTORY_MODEL = 'App\VoucherState'; // the related model to store the history @@ -156,7 +163,7 @@ private static function getContainingRange($start, $end, array $ranges): ?object if ($start <= $end && // query is properly formed $start >= $range->start && // our start is gte range start $end <= $range->end // our end is lte range end - ) { + ) { // early return on success return $range; } @@ -180,14 +187,14 @@ public static function createRangeDefFromVoucherCodes($startCode, $endCode) // Slightly complicated way of making an object that represents the range. // Destructure the output of into an assoc array - [ 'shortcode' => $rangeDef['shortcode'], 'number' => $rangeDef['start'] ] = self::splitShortcodeNumeric($startCode); - [ 'number' => $rangeDef['end'] ] = self::splitShortcodeNumeric($endCode); + ['shortcode' => $rangeDef['shortcode'], 'number' => $rangeDef['start']] = self::splitShortcodeNumeric($startCode); + ['number' => $rangeDef['end']] = self::splitShortcodeNumeric($endCode); // Modify the start/end numbers to integers $rangeDef["start"] = intval($rangeDef["start"]); $rangeDef["end"] = intval($rangeDef["end"]); - return (object) $rangeDef; + return (object)$rangeDef; } /** @@ -218,7 +225,7 @@ public static function splitShortcodeNumeric(string $code) $matches = []; // split into named matche and return if (preg_match("/^(?\D*)(?\d+)$/", $code, $matches) == 1) { - return $matches; + return $matches; } else { return false; } @@ -367,11 +374,11 @@ public function delivery(): BelongsTo * * @return HasOne */ - public function paymentPendedOn() + public function paymentPendedOn(): HasOne { return $this->hasOne(VoucherState::class) - ->where('to', 'payment_pending') - ->orderBy('created_at', 'desc'); + ->where('to', 'payment_pending') + ->orderBy('created_at', 'desc'); } /** @@ -380,11 +387,11 @@ public function paymentPendedOn() * * @return HasOne */ - public function recordedOn() + public function recordedOn(): HasOne { return $this->hasOne(VoucherState::class) - ->where('to', 'recorded') - ->orderBy('created_at', 'desc'); + ->where('to', 'recorded') + ->orderBy('created_at', 'desc'); } /** @@ -393,11 +400,36 @@ public function recordedOn() * * @return HasOne */ - public function reimbursedOn() + public function reimbursedOn(): HasOne { return $this->hasOne(VoucherState::class) - ->where('to', 'reimbursed') - ->orderBy('created_at', 'desc'); + ->where('to', 'reimbursed') + ->orderBy('created_at', 'desc'); + } + + public function rvid(): ?string + { + $centreSequence = $this->bundle?->registration->family->centre_sequence; + $centrePrefix = $this->bundle?->registration->family->initialCentre->prefix; + if ($centreSequence) { + return $centrePrefix . str_pad($centreSequence, 4, "0", STR_PAD_LEFT); + } + + return null; + } + + /** + * @return bool + */ + public function voucherHasBeenResurrected(): bool + { + $vs = $this->history()->get()->last(); + if ($vs) { + return $vs->to != "reimbursed"; + } + + // ???? can a voucher have no state? + return false; } /** @@ -431,13 +463,13 @@ public static function findByCodes($codes) return self::whereIn('code', $codes)->get(); } - /** - * Retrieve the min and max paymentPendedOn date of a collection of vouchers. - * - * @param Collection $vouchers - * - * @return array - */ + /** + * Retrieve the min and max paymentPendedOn date of a collection of vouchers. + * + * @param Collection $vouchers + * + * @return array + */ public static function getMinMaxVoucherDates(Collection $vouchers) { $sorted_vouchers = $vouchers->sortBy(function ($voucher) { @@ -481,33 +513,52 @@ public function scopeInDefinedRange($query, $rangeDef) public function scopeInOneOfStates(Builder $query, $states) { return $query - ->whereIn('currentstate', $states) - ; + ->whereIn('currentstate', $states); } /** * Gets a set of vouchers that are voidable. * * @param Builder $query - * @param array $states * @return Builder */ public function scopeInVoidableState(Builder $query) { $voidable_states = ['dispatched']; return $query - ->inOneOfStates($voidable_states) - ; + ->inOneOfStates($voidable_states); } /** * Prepare a date for array / JSON serialization. * - * @param \DateTimeInterface $date + * @param \DateTimeInterface $date * @return string */ protected function serializeDate(DateTimeInterface $date) { return $date->format('Y-m-d H:i:s'); } + + public function deepExport(): array + { + return [ + $this->code, + $this->sponsor?->name, + $this->delivery?->dispatched_at->format("Y/m/d"), + $this->delivery?->centre->name, + $this->delivery?->centre->sponsor->name, + $this->bundle?->disbursed_at ? "True" : "False", + $this->bundle?->disbursed_at?->format("Y/m/d"), + (string)$this->rvid(), + $this->bundle?->registration->family->carers[0]->name, + $this->bundle?->registration->centre->name, + $this->recordedOn->created_at->format("Y/m/d"), + $this->trader->name, + $this->trader->market->name, + $this->trader->market->sponsor->name, + $this->paymentPendedOn->created_at->format("Y/m/d"), + $this->reimbursedOn->created_at->format("Y/m/d"), + ]; + } } diff --git a/app/VoucherState.php b/app/VoucherState.php index adec99b5..a05e2985 100644 --- a/app/VoucherState.php +++ b/app/VoucherState.php @@ -4,6 +4,7 @@ use Eloquent; use Illuminate\Database\Eloquent\Model; +use Illuminate\Support\Carbon; // hard deletes on these; if only because we'll data-warehouse them at some point. @@ -15,6 +16,8 @@ * @property Voucher $voucher; * @property User $user; * @property StateToken $stateToken; + * @property Carbon $created_at; + * @property Carbon $updated_at; * * Notre sure what these are? 'user_type', 'source', */ diff --git a/bin/export-all-in-years.sh b/bin/export-all-in-years.sh new file mode 100755 index 00000000..bf669b44 --- /dev/null +++ b/bin/export-all-in-years.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +ARTISAN=$(dirname $0)/../artisan +$ARTISAN arc:mvl:export --chunk-size=999950 --to=30/03/2019 +$ARTISAN arc:mvl:export --chunk-size=999950 --from=01/04/2019 --to=30/03/2020 +$ARTISAN arc:mvl:export --chunk-size=999950 --from=01/04/2020 --to=30/03/2021 +$ARTISAN arc:mvl:export --chunk-size=999950 --from=01/04/2021 --to=30/03/2022 +$ARTISAN arc:mvl:export --chunk-size=999950 --from=01/04/2022 --to=30/03/2023 diff --git a/bin/process.sh b/bin/process.sh new file mode 100755 index 00000000..c5041f17 --- /dev/null +++ b/bin/process.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +ARTISAN=$(dirname $0)/../artisan +TARGET_DIR=$1 + +if [ -z $1 ]; then + echo Dir not found + exit 1 +fi + +for x in $(ls $1/*.arcx); do + $ARTISAN arc:mvl:process $x +done diff --git a/database/seeders/LargeVouchersSeeder.php b/database/seeders/LargeVouchersSeeder.php index 4f57c197..d7bc9b60 100644 --- a/database/seeders/LargeVouchersSeeder.php +++ b/database/seeders/LargeVouchersSeeder.php @@ -9,6 +9,7 @@ use App\Voucher; use App\VoucherState; use Carbon\Carbon; +use Faker\Factory; use Illuminate\Database\Seeder; use Illuminate\Support\Facades\Auth; @@ -87,5 +88,20 @@ public function run(): void // Make a transition definition $reimbursedTransitionDef = Voucher::createTransitionDef("payment_pending", "payout"); VoucherState::batchInsert($reimbursedVouchers, $date, 1, 'User', $reimbursedTransitionDef); + + // To make the vouchers usable to test MVL export we need random created dates but batch insert forces the date + // to be now + // THIS DOES NOT SEEM TO WORK, use this SQL instead see docs/README.md + /* + $faker = Factory::create(); + $voucherStates = VoucherState::where("to", "=", "reimbursed")->get(); + print "Got vouchers " . count($voucherStates) . "\n"; + foreach ($voucherStates as $vs) { + $createdAt = $faker->dateTimeBetween("-3 years")->format('Y-m-d H:i:s'); + $vs->created_at = $createdAt; + print "Voucher " . $vs->id . ", date: " . $createdAt . "\n"; + $vs->save(); + } + */ } } diff --git a/docs/ERD.png b/docs/ERD.png new file mode 100644 index 00000000..0b96b3c4 Binary files /dev/null and b/docs/ERD.png differ diff --git a/docs/MVL-EXPORT.md b/docs/MVL-EXPORT.md new file mode 100644 index 00000000..18c37714 --- /dev/null +++ b/docs/MVL-EXPORT.md @@ -0,0 +1,37 @@ +# Master Voucher Log Export + +As voucher number grow above 3x10^6 the exporting of full voucher details is taking a long time. To allow this to be spit into smaller chunks the MVL export process can now be run in smaller units. + +There are two commands, the first creates a list of vouchers that are in the state `reimbursed` and the second pulls their full data from the DB. The export process takes minutes but the processing takes up to around 20 seconds for 1000 vouchers. + +## Exporting a voucher list + +This will export all vouchers with a current state of reimbursed that were reimbursed in financial year 21/22. The vouchers will be placed into sequentially numbered files with no more than 50,000 vouches on each file. + +```bash +./artisan arc:mvl:export --chunk-size=50000 --from=01/04/2021 --to=31/03/2022 +``` + +This process those files. This example is the first chunk produced by the previous command. + +```bash +./artisan arc:mvl:process /home/tobias/usr/arc/service/storage/app/local/mvl/export/2023-10-10/vouchers.20210401-to-20220331.0000.txt +``` + +There are a couple of helper `bash` scripts to help this [here](../bin). + +## Export all years + +Does what it says, it exports data into one file for each financial year up to 22/23. + +## Process + +This takes a directory as input and processes all `.arcx` files in that directory into `.csv` files with a [deep export](https://github.com/neontribe/ARCVService/blob/develop/app/Voucher.php#:~:text=) row of fields. It also creates a file called `_headers.csv` that has the csv row headers. + +After running the process commad you can concatonate the csv files into a single file using: + +```bash +cat *.csv > /path/to/dest/FILENAME.csv +``` + +**N.B.** These csv files are processed using excel so may not contain more than 1,000,000 rows. \ No newline at end of file diff --git a/docs/README.md b/docs/README.md index 32f128b2..6a8a433b 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,14 +1,37 @@ -## Using Plant UML to generate diagrams -### Install plantuml - * Native: `sudo apt install plantuml` - * IntelliJ: https://plugins.jetbrains.com/plugin/7017-plantuml-integration - * VSCode: https://marketplace.visualstudio.com/items?itemName=jebbs.plantuml +## Links -### Create png from plantuml spec + * [Using Plant UML to generate diagrams](plantuml.md) -```bash -plantuml docs/voucher-state-transitions.txt -``` +## Voucher state transitions +Defined in [voucher-state-transitions.txt](voucher-state-transitions.txt) + +![Transition table](voucher-state-transitions.png "Voucher transitions") + +## Setting up vouchers for testing MVL + +The seeder and the [VoucherState::batchInsert](https://github.com/neontribe/ARCVService/blob/84ec961bc7074c0aff1f6f2a09311a2ad6d9c94e/app/VoucherState.php#L49) overrides the created date with `now` so all our voucher states are clumped in a single day. + +You need to run some sql to do that. + +### Create the stored procedure + +If you are using the [dev-helper](https://github.com/neontribe/ARCVInfra/tree/main/docker/dev-helper) docker stack: + + mysql -ulamp -plamp -P3336 -h127.0.0.1 lamp < docs/create-randomise-voucher-states.sql + +Otherwise, you'll need to connect to mysql and run it yourself. I think homestead users can do this, but I'm not sure, please update this document if this is wrong: + + mysql -uhomestead -psecret -P3306 -h127.0.0.1 homestead < docs/create-randomise-voucher-states.sql + +### Run it + +Just connect to the mysql instance and call it, dev help version: + + mysql -ulamp -plamp -P3336 -h127.0.0.1 lamp -e "CALL UpdateRandomTimestamps();" + +And then check it worked: + + select count(*), created_at from voucher_states where `to` = 'reimbursed' group by created_at limit 30; diff --git a/docs/create-randomise-voucher-states.sql b/docs/create-randomise-voucher-states.sql new file mode 100644 index 00000000..f16245cd --- /dev/null +++ b/docs/create-randomise-voucher-states.sql @@ -0,0 +1,35 @@ +DELIMITER // +CREATE PROCEDURE UpdateRandomTimestamps() +BEGIN + DECLARE done INT DEFAULT FALSE; + DECLARE pk_var INT; + DECLARE random_timestamp TIMESTAMP; + + DECLARE cur CURSOR FOR + SELECT id FROM voucher_states; + + DECLARE CONTINUE HANDLER FOR NOT FOUND SET done = TRUE; + + OPEN cur; + + read_loop: LOOP + FETCH cur INTO pk_var; + IF done THEN + LEAVE read_loop; + END IF; + + SET random_timestamp = TIMESTAMPADD(SECOND, + FLOOR(RAND() * + TIMESTAMPDIFF(SECOND, '2023-01-01', NOW())), + '2023-01-01'); + + UPDATE voucher_states + SET created_at = random_timestamp + WHERE id = pk_var; + END LOOP; + + CLOSE cur; +END // +DELIMITER ; + +CALL UpdateRandomTimestamps(); \ No newline at end of file diff --git a/docs/plantuml.md b/docs/plantuml.md new file mode 100644 index 00000000..32f128b2 --- /dev/null +++ b/docs/plantuml.md @@ -0,0 +1,14 @@ +## Using Plant UML to generate diagrams + +### Install plantuml + + * Native: `sudo apt install plantuml` + * IntelliJ: https://plugins.jetbrains.com/plugin/7017-plantuml-integration + * VSCode: https://marketplace.visualstudio.com/items?itemName=jebbs.plantuml + +### Create png from plantuml spec + +```bash +plantuml docs/voucher-state-transitions.txt +``` + diff --git a/tests/Console/Commands/MvlExportTest.php b/tests/Console/Commands/MvlExportTest.php new file mode 100644 index 00000000..bf5c6ecd --- /dev/null +++ b/tests/Console/Commands/MvlExportTest.php @@ -0,0 +1,37 @@ +make('Illuminate\Contracts\Console\Kernel')->bootstrap(); + return $app; + } + + public function testParameters(): void + { + $this->artisan('arc:mvl:export') + ->assertExitCode(0) + ->expectsOutput('Starting voucher export from 1970/01/01 to 2023/08/31 in chunks of 900000.'); + + $params = [ + "--from" => "01/04/2022", + "--to" => "31/03/2023", + "--chunk-size" => "54321", + ]; + $this->withoutMockingConsoleOutput()->artisan("arc:mvl:export", $params); + $result = Artisan::output(); + $this->assertEquals( + "Starting voucher export from 2022/04/01 to 2023/03/31 in chunks of 54321.\n", + $result + ); + print $result; + } + +}