Skip to content

Commit

Permalink
Merge pull request #9 from orisintel/feature/include-data
Browse files Browse the repository at this point in the history 
toggle to include data in dump
  • Loading branch information
Tom Schlick authored Sep 27, 2019
2 parents 4312ace + 53d86d3 commit a7d019f
Show file tree
Hide file tree
Showing 3 changed files with 273 additions and 60 deletions.
12 changes: 12 additions & 0 deletions config/migration-snapshot.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,4 +44,16 @@
|
*/
'trim-underscores' => env('MIGRATION_SNAPSHOT_TRIM_UNDERSCORES', true),

/*
|--------------------------------------------------------------------------
| Include Data
|--------------------------------------------------------------------------
|
| Include existing table data in the database dump. Useful for when you
| have constant defined values like a system user with a specific ID or
| records with special IDs which must match another environment.
|
*/
'data' => env('MIGRATION_SNAPSHOT_DATA', false),
];
239 changes: 201 additions & 38 deletions src/Commands/MigrateDumpCommand.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,23 +2,30 @@

namespace OrisIntel\MigrationSnapshot\Commands;

final class MigrateDumpCommand extends \Illuminate\Console\Command
use Illuminate\Console\Command;
use Illuminate\Support\Facades\DB;

final class MigrateDumpCommand extends Command
{
public const SCHEMA_SQL_PATH_SUFFIX = '/migrations/sql/schema.sql';
public const DATA_SQL_PATH_SUFFIX = '/migrations/sql/data.sql';

public const SUPPORTED_DB_DRIVERS = ['mysql', 'pgsql', 'sqlite'];

protected $signature = 'migrate:dump
{--database= : The database connection to use}';
{--database= : The database connection to use}
{--include-data : Include data present in the tables that was created via migrations. }
';

protected $description = 'Dump current database schema/structure as plain-text SQL file.';

public function handle()
{
$exit_code = null;

$database = $this->option('database') ?: \DB::getDefaultConnection();
\DB::setDefaultConnection($database);
$db_config = \DB::getConfig();
$database = $this->option('database') ?: DB::getDefaultConnection();
DB::setDefaultConnection($database);
$db_config = DB::getConfig();

// CONSIDER: Ending with ".mysql" or "-mysql.sql" unless in
// compatibility mode.
Expand All @@ -41,7 +48,7 @@ public function handle()
// CONSIDER: Option to dump to console Stdout instead.
// CONSIDER: Option to dump for each DB connection instead of only one.
// CONSIDER: Separate classes.
$method = $db_config['driver'] . 'Dump';
$method = $db_config['driver'] . 'SchemaDump';
$exit_code = self::{$method}($db_config, $schema_sql_path);

if (0 !== $exit_code) {
Expand All @@ -54,8 +61,34 @@ public function handle()
}

$this->info('Dumped schema');

if (! $this->option('include-data')) {
return;
}

$this->info('Starting Data Dump');

$data_sql_path = database_path() . self::DATA_SQL_PATH_SUFFIX;

$method = $db_config['driver'] . 'DataDump';
$exit_code = self::{$method}($db_config, $data_sql_path);

if (0 !== $exit_code) {
if (file_exists($data_sql_path)) {
unlink($data_sql_path);
}

exit($exit_code);
}

$this->info('Dumped Data');
}

/**
* @param array $output
*
* @return array
*/
public static function reorderMigrationRows(array $output) : array
{
if (config('migration-snapshot.reorder')) {
Expand Down Expand Up @@ -95,31 +128,18 @@ public static function reorderMigrationRows(array $output) : array
*
* @return int containing exit code.
*/
private static function mysqlDump(array $db_config, string $schema_sql_path) : int
private static function mysqlSchemaDump(array $db_config, string $schema_sql_path) : int
{
// CONSIDER: Supporting unix_socket.
// CONSIDER: Alternative tools like `xtrabackup` or even just querying
// "SHOW CREATE TABLE" via Eloquent.
// CONSIDER: Capturing Stderr and outputting with `$this->error()`.

// Not including connection name in file since typically only one DB.
// Excluding any hash or date suffix since only current is relevant.
$command_prefix = 'mysqldump --routines --skip-add-drop-table'
. ' --skip-add-locks --skip-comments --skip-set-charset --tz-utc'
. ' --host=' . escapeshellarg($db_config['host'])
. ' --port=' . escapeshellarg($db_config['port'])
. ' --user=' . escapeshellarg($db_config['username'])
. ' --password=' . escapeshellarg($db_config['password'])
. ' ' . escapeshellarg($db_config['database']);
// TODO: Suppress warning about insecure password.
// CONSIDER: Intercepting stdout and stderr and converting to colorized
// console output with `$this->info` and `->error`.
passthru(
$command_prefix
static::mysqlCommandPrefix($db_config)
. ' --result-file=' . escapeshellarg($schema_sql_path)
. ' --no-data',
$exit_code
);

if (0 !== $exit_code) {
return $exit_code;
}
Expand All @@ -128,6 +148,7 @@ private static function mysqlDump(array $db_config, string $schema_sql_path) : i
if (false === $schema_sql) {
return 1;
}

$schema_sql = preg_replace('/\s+AUTO_INCREMENT=[0-9]+/iu', '', $schema_sql);
$schema_sql = self::trimUnderscoresFromForeign($schema_sql);
if (false === file_put_contents($schema_sql_path, $schema_sql)) {
Expand All @@ -136,10 +157,11 @@ private static function mysqlDump(array $db_config, string $schema_sql_path) : i

// Include migration rows to avoid unnecessary reruns conflicting.
exec(
$command_prefix . ' migrations --no-create-info --skip-extended-insert --compact',
static::mysqlCommandPrefix($db_config) . ' migrations --no-create-info --skip-extended-insert --compact',
$output,
$exit_code
);

if (0 !== $exit_code) {
return $exit_code;
}
Expand All @@ -157,6 +179,63 @@ private static function mysqlDump(array $db_config, string $schema_sql_path) : i
return $exit_code;
}

/**
* @param array $db_config like ['host' => , 'port' => ].
* @param string $data_sql_path like '.../data.sql'
*
* @return int containing exit code.
*/
private static function mysqlDataDump(array $db_config, string $data_sql_path) : int
{
passthru(
static::mysqlCommandPrefix($db_config)
. ' --result-file=' . escapeshellarg($data_sql_path)
. ' --no-create-info --skip-triggers'
. ' --ignore-table=' . escapeshellarg($db_config['database'] . '.migrations'),
$exit_code
);

if (0 !== $exit_code) {
return $exit_code;
}

$data_sql = file_get_contents($data_sql_path);
if (false === $data_sql) {
return 1;
}

$data_sql = preg_replace('/\s+AUTO_INCREMENT=[0-9]+/iu', '', $data_sql);
if (false === file_put_contents($data_sql_path, $data_sql)) {
return 1;
}

return $exit_code;
}

/**
* @param array $db_config
*
* @return string
*/
private static function mysqlCommandPrefix(array $db_config) : string
{
// CONSIDER: Supporting unix_socket.
// CONSIDER: Alternative tools like `xtrabackup` or even just querying
// "SHOW CREATE TABLE" via Eloquent.
// CONSIDER: Capturing Stderr and outputting with `$this->error()`.

// Not including connection name in file since typically only one DB.
// Excluding any hash or date suffix since only current is relevant.

return 'mysqldump --routines --skip-add-drop-table'
. ' --skip-add-locks --skip-comments --skip-set-charset --tz-utc'
. ' --host=' . escapeshellarg($db_config['host'])
. ' --port=' . escapeshellarg($db_config['port'])
. ' --user=' . escapeshellarg($db_config['username'])
. ' --password=' . escapeshellarg($db_config['password'])
. ' ' . escapeshellarg($db_config['database']);
}

/**
* Trim underscores from FK constraint names to workaround PTOSC quirk.
*
Expand Down Expand Up @@ -208,38 +287,30 @@ public static function trimUnderscoresFromForeign(string $sql) : string

/**
* @param array $db_config like ['host' => , 'port' => ].
* @param string $schema_sql_path
*
* @return int containing exit code.
*/
private static function pgsqlDump(array $db_config, string $schema_sql_path) : int
private static function pgsqlSchemaDump(array $db_config, string $schema_sql_path) : int
{
// CONSIDER: Supporting unix_socket.
// CONSIDER: Instead querying pg catalog tables via Eloquent.
// CONSIDER: Capturing Stderr and outputting with `$this->error()`.

// CONSIDER: Instead using DSN-like URL instead of env. var. for pass.
$command_prefix = 'PGPASSWORD=' . escapeshellarg($db_config['password'])
. ' pg_dump'
. ' --host=' . escapeshellarg($db_config['host'])
. ' --port=' . escapeshellarg($db_config['port'])
. ' --username=' . escapeshellarg($db_config['username'])
. ' ' . escapeshellarg($db_config['database']);
passthru(
$command_prefix
static::pgsqlCommandPrefix($db_config)
. ' --file=' . escapeshellarg($schema_sql_path)
. ' --schema-only',
$exit_code
);

if (0 !== $exit_code) {
return $exit_code;
}

// Include migration rows to avoid unnecessary reruns conflicting.
exec(
$command_prefix . ' --table=migrations --data-only --inserts',
static::pgsqlCommandPrefix($db_config) . ' --table=migrations --data-only --inserts',
$output,
$exit_code
);

if (0 !== $exit_code) {
return $exit_code;
}
Expand All @@ -264,13 +335,51 @@ function ($line) {
return $exit_code;
}

/**
* @param array $db_config
* @param string $data_sql_path
*
* @return int
*/
private static function pgsqlDataDump(array $db_config, string $data_sql_path) : int
{
passthru(
static::pgsqlCommandPrefix($db_config)
. ' --file=' . escapeshellarg($data_sql_path)
. ' --exclude-table=' . escapeshellarg($db_config['database'] . '.migrations')
. ' --data-only',
$exit_code
);

if (0 !== $exit_code) {
return $exit_code;
}

return $exit_code;
}

/**
* @param array $db_config
*
* @return string
*/
private static function pgsqlCommandPrefix(array $db_config) : string
{
return 'PGPASSWORD=' . escapeshellarg($db_config['password'])
. ' pg_dump'
. ' --host=' . escapeshellarg($db_config['host'])
. ' --port=' . escapeshellarg($db_config['port'])
. ' --username=' . escapeshellarg($db_config['username'])
. ' ' . escapeshellarg($db_config['database']);
}

/**
* @param array $db_config like ['host' => , 'port' => ].
* @param string $schema_sql_path like '.../schema.sql'
*
* @return int containing exit code.
*/
private static function sqliteDump(array $db_config, string $schema_sql_path) : int
private static function sqliteSchemaDump(array $db_config, string $schema_sql_path) : int
{
// CONSIDER: Accepting command name as option or from config.
$command_prefix = 'sqlite3 ' . escapeshellarg($db_config['database']);
Expand All @@ -283,6 +392,7 @@ private static function sqliteDump(array $db_config, string $schema_sql_path) :
if (0 !== $exit_code) {
return $exit_code;
}

$tables = preg_split('/\s+/', implode(' ', $output));

file_put_contents($schema_sql_path, '');
Expand All @@ -297,6 +407,7 @@ private static function sqliteDump(array $db_config, string $schema_sql_path) :
$output,
$exit_code
);

if (0 !== $exit_code) {
return $exit_code;
}
Expand All @@ -316,4 +427,56 @@ private static function sqliteDump(array $db_config, string $schema_sql_path) :

return $exit_code;
}

/**
* @param array $db_config
* @param string $data_sql_path
*
* @return int
*/
private static function sqliteDataDump(array $db_config, string $data_sql_path) : int
{
// CONSIDER: Accepting command name as option or from config.
$command_prefix = 'sqlite3 ' . escapeshellarg($db_config['database']);

// Since Sqlite lacks Information Schema, and dumping everything may be
// too slow or memory intense, just query tables and dump them
// individually.
// CONSIDER: Using Laravel's `Schema` code instead.
exec($command_prefix . ' .tables', $output, $exit_code);
if (0 !== $exit_code) {
return $exit_code;
}

$tables = preg_split('/\s+/', implode(' ', $output));

foreach ($tables as $table) {
// We don't want to dump the migrations table here
if ('migrations' === $table) {
continue;
}

// Only migrations should dump data with schema.
$sql_command = '.dump';

$output = [];
exec(
$command_prefix . ' ' . escapeshellarg("$sql_command $table"),
$output,
$exit_code
);

if (0 !== $exit_code) {
return $exit_code;
}

file_put_contents(
$data_sql_path,
implode(PHP_EOL, $output) . PHP_EOL,
FILE_APPEND
);
}

return $exit_code;
}
}
Loading

0 comments on commit a7d019f

Please sign in to comment.