From fffe5a9068eb6ee49c19b17850a00304c6f07da4 Mon Sep 17 00:00:00 2001 From: Pierre Rineau Date: Mon, 27 May 2024 17:00:48 +0200 Subject: [PATCH] no issue - configuration changes and old names deprecation --- CHANGELOG.md | 14 + config/packages/db_tools.yaml | 10 +- docs/.vitepress/config.ts | 1 + docs/content/configuration.md | 28 +- docs/content/configuration/reference.md | 469 +++++++++++------- docs/content/console.md | 94 ++++ src/Bridge/Standalone/Bootstrap.php | 64 ++- .../Standalone/StandaloneConfiguration.php | 6 - .../DbToolsConfiguration.php | 62 ++- .../DependencyInjection/DbToolsExtension.php | 63 ++- .../Symfony/Resources/config/services.yaml | 57 ++- src/Helper/Cli/Bootstrap.php | 466 ----------------- src/Helper/Cli/Context.php | 31 -- src/Helper/Cli/StandaloneConfiguration.php | 57 --- src/Storage/Storage.php | 2 +- .../config/packages/db_tools_alt1.yaml | 13 +- .../config/packages/db_tools_alt2.yaml | 10 +- .../DbToolsConfigurationTest.php | 135 +++-- 18 files changed, 712 insertions(+), 870 deletions(-) create mode 100644 docs/content/console.md delete mode 100644 src/Helper/Cli/Bootstrap.php delete mode 100644 src/Helper/Cli/Context.php delete mode 100644 src/Helper/Cli/StandaloneConfiguration.php diff --git a/CHANGELOG.md b/CHANGELOG.md index 09964c4c..768cfe14 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,12 +2,26 @@ ## 1.2.0 +* [feature] ⭐️ Add new standlone console/CLI tool running outside of the Symfony context, + this allows this bundle to run outside of a Symfony application on top of any other framework (#153). +* [feature] ⭐️ Add experimental PHAR compiler (#155). +* [deprecation] ⚠️ `excluded_tables` configuration option is deprecated, use `backup_excluded_tables` instead. +* [deprecation] ⚠️ `backupper_binaries` configuration option is deprecated, use `backup_binaries` instead. +* [deprecation] ⚠️ `restorer_binaries` configuration option is deprecated, use `restore_binaries` instead. +* [deprecation] ⚠️ `backupper_options` configuration option is deprecated, use `backup_options` instead. +* [deprecation] ⚠️ `restorer_options` configuration option is deprecated, use `restore_options` instead. +* [deprecation] ⚠️ `MakinaCorpus\DbToolsBundle\DbToolsBundle` class is renamed, please use `MakinaCorpus\DbToolsBundle\Bridge\Symfony\DbToolsBundle` instead. +* [removal] ⚠️ `storage_directory` configuration option is removed, use `storage.root_dir` instead. + +## 1.2.0 + * [feature] ⭐️ Add Doctrine DBAL 4.0 compatibility (#140). * [feature] ⭐️ Add Doctrine ORM 3.0 compatibility as a side effect of Doctrine DBAL 4.0 support (#140). * [feature] ⭐️ Anonymization - Add Doctrine Embeddables support (#105). * [feature] ⭐️ Anonymization - Add Doctrine entity joined inheritance support (#160) * [feature] ⭐️ Anonymization - Finalized and improved IBAN/BIC anonymizer (#4) * [fix] Restored MySQL 5.7 support (#124) +* [deprecation] ⚠️ `storage_directory` configuration option is deprecated, use `storage.root_dir` instead. * [internal] Remove `doctrine/dbal` dependency from all code except the database session registry (#142). * [internal] Introduce `DatabaseSessionRegistry` as single entry point for plugging-in database (#142). * [internal] Use `makinacorpus/query-builder` schema manager for DDL alteration (#140). diff --git a/config/packages/db_tools.yaml b/config/packages/db_tools.yaml index ef12dd7f..990c7266 100644 --- a/config/packages/db_tools.yaml +++ b/config/packages/db_tools.yaml @@ -35,17 +35,17 @@ db_tools: # restore_timeout: 2400 # default 1800 # List here tables (per connection) you don't want in your backups - #excluded_tables: + #backup_excluded_tables: #default: ['table1', 'table2'] # Specify here paths to binaries, only if the system can't find them by himself # platform are 'mysql', 'postgresql', 'sqlite' - #backupper_binaries: + #backup_binaries: #mariadb: '/usr/bin/mariadb-dump' # default 'mariadb-dump' #mysql: '/usr/bin/mysqldump' # default 'mysqldump' #postgresql: '/usr/bin/pg_dump' # default 'pg_dump' #sqlite: '/usr/bin/sqlite3' # default 'sqlite3' - #restorer_binaries: + #restore_binaries: #mariadb: '/usr/bin/mariadb' # default 'mariadb' #mysql: '/usr/bin/mysql' # default 'mysql' #postgresql: '/usr/bin/pg_restore' # default 'pg_restore' @@ -66,10 +66,10 @@ db_tools: # - MySQL: None # - PostgreSQL: -j 2 --clean --if-exists --disable-triggers # - SQLite: None - #backupper_options: + #backup_options: #default: '' #another_connection: '' - #restorer_options: + #restore_options: #default: '' #another_connection: '' diff --git a/docs/.vitepress/config.ts b/docs/.vitepress/config.ts index adb9b897..37e49d02 100644 --- a/docs/.vitepress/config.ts +++ b/docs/.vitepress/config.ts @@ -61,6 +61,7 @@ export default defineConfig({ { text: 'Installation', link: '/getting-started/installation' }, { text: 'Basics', link: '/getting-started/basics' }, { text: 'Supported databases', link: '/getting-started/database-vendors' }, + { text: 'CLI tool', link: '/console' }, ] }, { diff --git a/docs/content/configuration.md b/docs/content/configuration.md index 0967c7df..343db228 100644 --- a/docs/content/configuration.md +++ b/docs/content/configuration.md @@ -91,7 +91,7 @@ This will allow the restore command to find your backups. ### Excluded tables -The `excluded_tables` parameter let you configure tables to exclude from backups. You will need to give a +The `backup_excluded_tables` parameter let you configure tables to exclude from backups. You will need to give a configuration per doctrine connection. Default value is `null`: no table are excluded. @@ -105,7 +105,7 @@ db_tools: #... - excluded_tables: + backup_excluded_tables: default: ['table1', 'table2'] #... @@ -198,16 +198,16 @@ If the `db-tools:check` command returns you some errors: # Specify here paths to binaries, only if the system can't find them by himself # platform are 'mysql', 'postgresql', 'sqlite' - backupper_binaries: - mariadb: '/usr/bin/mariadb-dump' # default 'mariadb-dump' - mysql: '/usr/bin/mysqldump' # default 'mysqldump' - postgresql: '/usr/bin/pg_dump' # default 'pg_dump' - sqlite: '/usr/bin/sqlite3' # default 'sqlite3' - restorer_binaries: - mariadb: '/usr/bin/mariadb' # default 'mariadb' - mysql: '/usr/bin/mysql' # default 'mysql' - postgresql: '/usr/bin/pg_restore' # default 'pg_restore' - sqlite: '/usr/bin/sqlite3' # default 'sqlite3' + backup_binaries: + mariadb: '/usr/bin/mariadb-dump' + mysql: '/usr/bin/mysqldump' + postgresql: '/usr/bin/pg_dump' + sqlite: '/usr/bin/sqlite3' + restore_binaries: + mariadb: '/usr/bin/mariadb' + mysql: '/usr/bin/mysql' + postgresql: '/usr/bin/pg_restore' + sqlite: '/usr/bin/sqlite3' #... ``` @@ -250,10 +250,10 @@ by configuring your own ones per operation type and DBAL connection: db_tools: # ... - backupper_options: + backup_options: default: '--an-option' another_connection: '-xyz --another' - restorer_options: + restore_options: default: '--a-first-one --a-second-one' another_connection: '-O sample-value' ``` diff --git a/docs/content/configuration/reference.md b/docs/content/configuration/reference.md index 7f454c26..549813f1 100644 --- a/docs/content/configuration/reference.md +++ b/docs/content/configuration/reference.md @@ -11,200 +11,275 @@ In all cases, it requires a configuration file. When running throught the Symfony project console, configuration file is not required since it will auto-configure by reading your Symfony site configuration. -:::tip +**Environment related configuration can be set throught environment variables**, +it includes anything related to database binaries, timeout, and a few other +configuration options. +When this option is available, an _Environment_ tab in code sample will be displayed. + +In the opposite, anything related to business domain or application matters can +only be configured throught the configuration file. + +**Both Symfony bundle and standalone CLI tool will use the environment variables per default.** + +:::info +When working with the standalone console tool, all relative path are +relative to the `workdir` option. **If none provided, all paths are** +**relative to the configuration file directory**. +::: + +:::warning When configuring in Symfony you must add an extra `db_tools` top-level section in order to avoid conflicts with other bundles. When configuring for the standalone console tool, this extra top-level section must be omitted. ::: -:::warning -When working with the standalone console tool, all relative path are -relative to the `workdir` option. If none provided, then path are -relative to the configuration file directory the path is defined -within. -::: - ## All options [`anonymization.tables` (standalone)](#anonymization-tables) | [`anonymization.yaml`](#anonymization-yaml) | [`anonymizer_paths`](#anonymizer-paths) | +[`backup_binaries`](#backup-binaries) | +[`backup_excluded_tables`](#excluded-tables) | [`backup_expiration_age`](#backup-expiration-age) | +[`backup_options`](#backup-options) | [`backup_timeout`](#backup-timeout) | -[`backupper_binaries`](#backupper-binaries) | -[`backupper_options`](#backupper-options) | [`connections` (standalone)](#connections) | +[`connections` (standalone)](#connections) | +[`default_connection` (standalone)](#default-connection) | [`default_connection` (standalone)](#default-connection) | -[`excluded_tables`](#excluded-tables) | +[`restore_binaries`](#restore-binaries) | +[`restore_options`](#restore-options) | [`restore_timeout`](#restore-timeout) | -[`restorer_binaries`](#restorer-binaries) | -[`restorer_options`](#restorer-options) | [`storage.filename_strategy`](#storage-filename-strategy) | [`workdir` (standalone)](#workdir) ## Common options -### `storage.root_dir` +### `anonymizer_paths` -Root directory of the backup storage manager. Default filename strategy will -always use this folder as a root path. +PHP source folders in which custom anonymizer implementations will be looked-up. + +This allows you to write custom implementations and use it. + +Path are local filesystem arbitrary paths, and you are allowed to set any path. +A recursive file system iterator will lookup in those folders and find classes +that extend the `MakinaCorpus\DbToolsBundle\Anonymization\Anonymizer\AbstractAnonymizer` +class within, then register those as anonymizers. :::code-group ```yaml [Symfony] db_tools: - storage: - root_dir: "%kernel.root_dir%/var/db_tools" + anonymizer_paths: + - '%kernel.project_dir%/vendor/makinacorpus/db-tools-bundle/src/Anonymizer' + - '%kernel.project_dir%/src/Anonymization/Anonymizer' ``` ```yaml [Standalone] -storage: - root_dir: "./var/db_tools" + anonymizer_paths: + - './vendor/makinacorpus/db-tools-bundle/src/Anonymizer' + - './src/Anonymization/Anonymizer' ``` ::: -### `storage.filename_strategy` - -Key value pairs, keys are connection names, values can be either: -- `default`: let the tool decide, it is an alias to `datetime`. -- `datetime`: stores backups in split timestamp directory tree, such as: `/YYYY/MM/-.` +### `anonymization.yaml` -When used in a Symfony application, the strategy can be a service name registered in the -container. This service must implement `MakinaCorpus\DbToolsBundle\Storage\FilenameStrategyInterface`. -See [filename strategies documentation](../backup_restore) for more information. +List of YAML configuration file that contains table and their columns to +anonymize. -Example: +For configuration format please refer the [anonymizers documentation](../anonymization/core-anonymizers). :::code-group ```yaml [Symfony] +# Single connection. db_tools: - storage: - filename_strategy: - connection_one: datetime - connection_two: default - connection_three: app.my_filename_strategy - connection_four: App\DbTools\Storage\MyCustomFilenameStrategy + anonymizer: + yaml: '%kernel.project_dir%/config/anonymizations.yaml' + +# Multiple named connections. +db_tools: + anonymizer: + yaml: + - connection_one: '%kernel.project_dir%/config/anonymizations/connection_one.yaml' + - connection_two: '%kernel.project_dir%/config/anonymizations/connection_two.yaml' ``` ```yaml [Standalone] -storage: - filename_strategy: - connection_one: datetime - connection_two: default - connection_four: App\DbTools\Storage\MyCustomFilenameStrategy +# Single connection. +anonymizer: + yaml: './db_tools.anonymization.yaml' + +# Multiple named connections. +anonymizer: + yaml: + - connection_one: './db_tools.connection_one.anonymization.yaml' + - connection_two: './db_tools.connection_two.anonymization.yaml' ``` ::: -### `backup_expiration_age` +### `backup_binaries` -Backup file expiration time after which they get deleted when running -the `backup` or `clean` command. +Path to backup command in filesystem. -It uses a relative date interval format as documented in https://www.php.net/manual/en/datetime.formats.php#datetime.formats.relative +Defaults are the well known executable names without absolute file path, which should +work in most Linux distributions. -Example: :::code-group ```yaml [Symfony] db_tools: - backup_expiration_age: '6 months ago' + backup_binaries: + mariadb: /usr/bin/mariadb-dump + mysql: /usr/bin/mysqldump + postgresql: /usr/bin/pg_dump + sqlite: /usr/bin/sqlite3 ``` ```yaml [Standalone] -backup_expiration_age: '6 months ago' +backup_binaries: + mariadb: /usr/bin/mariadb-dump + mysql: /usr/bin/mysqldump + postgresql: /usr/bin/pg_dump + sqlite: /usr/bin/sqlite3 ``` -::: -### `backup_timeout` +```ini [Environment] +DBTOOLS_BACKUP_BINARY_MARIADB="/usr/bin/mariadb-dump" +DBTOOLS_BACKUP_BINARY_MYSQL="/usr/bin/mysqldump" +DBTOOLS_BACKUP_BINARY_POSTGRESQL="/usr/bin/pg_dump" +DBTOOLS_BACKUP_BINARY_SQLITE="/usr/bin/sqlite3" +``` +::: -Backup process timeout in seconds. +### `backup_excluded_tables` -It uses a relative date interval format as documented in https://www.php.net/manual/en/datetime.formats.php#datetime.formats.relative -or accepts a number of seconds as an integer value. +Tables excluded from backup. Example: :::code-group ```yaml [Symfony] -# As a date interval string. +# Single connection. db_tools: - backup_timeout: '2 minutes and 7 seconds' + backup_excluded_tables: ['table1', 'table2'] -# As a number of seconds. +# Multiple named connections. db_tools: - backup_timeout: 67 + backup_excluded_tables: + connection_one: ['table1', 'table2'] + connection_two: ['table1', 'table2'] ``` ```yaml [Standalone] -# As a date interval string. -backup_timeout: '2 minutes and 7 seconds' +# Single connection. +backup_excluded_tables: ['table1', 'table2'] -# As a number of seconds. -backup_timeout: 67 +# Multiple named connections. +backup_excluded_tables: + connection_one: ['table1', 'table2'] + connection_two: ['table1', 'table2'] ``` ::: -### `excluded_tables` +### `backup_expiration_age` -Tables excluded from backup. +Backup file expiration time after which they get deleted when running +the `backup` or `clean` command. + +It uses a relative date interval format as documented in https://www.php.net/manual/en/datetime.formats.relative.php Example: :::code-group ```yaml [Symfony] db_tools: - excluded_tables: - connection_one: ['table1', 'table2'] - connection_two: ['table1', 'table2'] + backup_expiration_age: '6 months ago' ``` -```yaml [Symfony alt.] +```yaml [Standalone] +backup_expiration_age: '6 months ago' +``` + +```ini [Environment] +DBTOOLS_BACKUP_EXPIRATION_AGE="6 months ago" +``` +::: + +### `backup_options` + +Allows you to add specific command line options to the backup command, one for each connection. + +If you do not define some default options, here or by using the "--extra-options" option when +invoking the command, the following ones will be used according to the database vendor: + - MariaDB: `--no-tablespaces` + - MySQL: `--no-tablespaces` + - PostgreSQL: `-Z 5 --lock-wait-timeout=120` + - SQLite: `-bail` + +By specifying options, the default ones will be dropped. + +:::code-group +```yaml [Symfony] db_tools: - # If you have a single connection. - excluded_tables: ['table1', 'table2'] + backup_options: + connection_one: '-Z 5 --lock-wait-timeout=120' + connection_two: '--no-tablespaces' ``` ```yaml [Standalone] -excluded_tables: - connection_one: ['table1', 'table2'] - connection_two: ['table1', 'table2'] +backup_options: + connection_one: '-Z 5 --lock-wait-timeout=120' + connection_two: '--no-tablespaces' ``` -```yaml [Standalone alt.] - # If you have a single connection. - excluded_tables: ['table1', 'table2'] +```ini [Environment] +# Only supports single connection. +DBTOOLS_BACKUP_OPTIONS="-Z 5 --lock-wait-timeout=120" ``` ::: +:::warning +Multiple connections configuration is not possible using environment variables yet. +::: -### `backupper_binaries` +### `backup_timeout` -Path to backup command in filesystem. +Backup process timeout in seconds. -Defaults are the well known executable names without absolute file path, which should -work in most Linux distributions. +It uses a relative date interval format as documented in https://www.php.net/manual/en/datetime.formats.php#datetime.formats.relative +or accepts a number of seconds as an integer value. +Example: :::code-group ```yaml [Symfony] +# As a date interval string. db_tools: - backupper_binaries: - mariadb: /usr/bin/mariadb-dump - mysql: /usr/bin/mysqldump - postgresql: /usr/bin/pg_dump - sqlite: /usr/bin/sqlite3 + backup_timeout: '2 minutes 7 seconds' + +# As a number of seconds. +db_tools: + backup_timeout: 67 ``` ```yaml [Standalone] -backupper_binaries: - mariadb: /usr/bin/mariadb-dump - mysql: /usr/bin/mysqldump - postgresql: /usr/bin/pg_dump - sqlite: /usr/bin/sqlite3 +# As a date interval string. +backup_timeout: '2 minutes 7 seconds' + +# As a number of seconds. +backup_timeout: 67 +``` + +```ini [Environment] +# As a date interval string. +DBTOOLS_BACKUP_TIMEOUT="2 minutes 7 seconds" + +# As a number of seconds. +DBTOOLS_BACKUP_TIMEOUT=67 ``` ::: -### `restorer_binaries` +### `restore_binaries` Path to restore command in filesystem. @@ -214,7 +289,7 @@ work in most Linux distributions. :::code-group ```yaml [Symfony] db_tools: - restorer_binaries: + restore_binaries: mariadb: /usr/bin/mariadb mysql: /usr/bin/mysql postgresql: /usr/bin/pg_restore @@ -222,42 +297,56 @@ db_tools: ``` ```yaml [Standalone] -restorer_binaries: +restore_binaries: mariadb: /usr/bin/mariadb mysql: /usr/bin/mysql postgresql: /usr/bin/pg_restore sqlite: /usr/bin/sqlite3 ``` + +```ini [Environment] +DBTOOLS_RESTORE_BINARY_MARIADB="/usr/bin/mariadb" +DBTOOLS_RESTORE_BINARY_MYSQL="/usr/bin/mysql" +DBTOOLS_RESTORE_BINARY_POSTGRESQL="/usr/bin/pg_restore" +DBTOOLS_RESTORE_BINARY_SQLITE="/usr/bin/sqlite3" +``` ::: -### `backupper_options` +### `restore_options` -Allows you to add specific command line options to the backup command, one for each connection. +Allows you to add specific command line options to the restore command, one for each connection. If you do not define some default options, here or by using the "--extra-options" option when invoking the command, the following ones will be used according to the database vendor: - - MariaDB: `--no-tablespaces` - - MySQL: `--no-tablespaces` - - PostgreSQL: `-Z 5 --lock-wait-timeout=120` - - SQLite: `-bail` - -By specifying options, the default ones will be dropped. + - MariaDB: None + - MySQL: None + - PostgreSQL: `-j 2 --clean --if-exists --disable-triggers` + - SQLite: None :::code-group ```yaml [Symfony] db_tools: - backupper_options: - connection_one: '-Z 5 --lock-wait-timeout=120' + restore_options: + connection_one: '-j 2 --clean --if-exists --disable-triggers' connection_two: '--no-tablespaces' ``` ```yaml [Standalone] -backupper_options: - connection_one: '-Z 5 --lock-wait-timeout=120' - connection_two: '--no-tablespaces' +restore_options: + connection_one: '-j 2 --clean --if-exists --disable-triggers' + connection_two: '--some-other-option +``` + +```ini [Environment] +# Only supports single connection. +DBTOOLS_RESTORE_OPTIONS="-j 2 --clean --if-exists --disable-triggers" ``` ::: +:::warning +Multiple connections configuration is not possible using environment variables yet. +::: + ### `restore_timeout` Restore process timeout in seconds. @@ -271,7 +360,7 @@ Example: ```yaml [Symfony] # As a date interval string. db_tools: - restore_timeout: '2 minutes and 7 seconds' + restore_timeout: '2 minutes 7 seconds' # As a number of seconds. db_tools: @@ -280,98 +369,92 @@ db_tools: ```yaml [Standalone] # As a date interval string. -restore_timeout: '2 minutes and 7 seconds' +restore_timeout: '2 minutes 7 seconds' # As a number of seconds. restore_timeout: 67 ``` -::: - -### `restorer_options` - -Allows you to add specific command line options to the restore command, one for each connection. - -If you do not define some default options, here or by using the "--extra-options" option when -invoking the command, the following ones will be used according to the database vendor: - - MariaDB: None - - MySQL: None - - PostgreSQL: `-j 2 --clean --if-exists --disable-triggers` - - SQLite: None -:::code-group -```yaml [Symfony] -db_tools: - backupper_options: - connection_one: '-j 2 --clean --if-exists --disable-triggers' - connection_two: '--no-tablespaces' -``` +```ini [Environment] +# As a date interval string. +DBTOOLS_RESTORE_TIMEOUT="2 minutes 7 seconds" -```yaml [Standalone] -backupper_options: - connection_one: '-j 2 --clean --if-exists --disable-triggers' - connection_two: '--some-other-option +# As a number of seconds. +DBTOOLS_RESTORE_TIMEOUT=67 ``` ::: -### `anonymizer_paths` +### `storage.filename_strategy` -PHP source folders in which custom anonymizer implementations will be looked-up. +Key value pairs, keys are connection names, values can be either: +- `default`: let the tool decide, it is an alias to `datetime`. +- `datetime`: stores backups in split timestamp directory tree, such as: `/YYYY/MM/-.` -This allows you to write custom implementations and use it. +When used in a Symfony application, the strategy can be a service name registered in the +container. This service must implement `MakinaCorpus\DbToolsBundle\Storage\FilenameStrategyInterface`. +See [filename strategies documentation](../backup_restore) for more information. -Path are local filesystem arbitrary paths, and you are allowed to set any path. -A recursive file system iterator will lookup in those folders and find classes -that extend the `MakinaCorpus\DbToolsBundle\Anonymization\Anonymizer\AbstractAnonymizer` -class within, then register those as anonymizers. +Example: :::code-group ```yaml [Symfony] +# Global default. db_tools: - anonymizer_paths: - - '%kernel.project_dir%/vendor/makinacorpus/db-tools-bundle/src/Anonymizer' - - '%kernel.project_dir%/src/Anonymization/Anonymizer' + storage: + filename_strategy: datetime + +# One per named connections. +db_tools: + storage: + filename_strategy: + connection_one: datetime + connection_two: default + connection_three: app.my_filename_strategy + connection_four: App\DbTools\Storage\MyCustomFilenameStrategy ``` ```yaml [Standalone] - anonymizer_paths: - - './vendor/makinacorpus/db-tools-bundle/src/Anonymizer' - - './src/Anonymization/Anonymizer' +# Global default. +storage: + filename_strategy: datetime + +# One per named connections. +storage: + filename_strategy: + connection_one: datetime + connection_two: default + connection_four: App\DbTools\Storage\MyCustomFilenameStrategy +``` + +```ini [Environment] +# Only supports global default. +DBTOOLS_STORAGE_FILENAME_STRATEGY=datetime ``` ::: -### `anonymization.yaml` +:::warning +Multiple connections configuration is not possible using environment variables yet. +::: -List of YAML configuration file that contains table and their columns to -anonymize. +### `storage.root_dir` -For configuration format please refer the [anonymizers documentation](../anonymization/core-anonymizers). +Root directory of the backup storage manager. Default filename strategy will +always use this folder as a root path. :::code-group ```yaml [Symfony] db_tools: - anonymizer: - yaml: - - connection_one: '%kernel.project_dir%/config/anonymizations/connection_one.yaml' - - connection_two: '%kernel.project_dir%/config/anonymizations/connection_two.yaml' -``` - -```yaml [Symfony alt.] -db_tools: - anonymizer: - # If you have a single connection. - yaml: '%kernel.project_dir%/config/anonymizations.yaml' + storage: + root_dir: "%kernel.root_dir%/var/db_tools" ``` ```yaml [Standalone] -anonymizer: - yaml: - - connection_one: './db_tools.connection_one.anonymization.yaml' - - connection_two: './db_tools.connection_two.anonymization.yaml' +storage: + root_dir: "./var/db_tools" ``` -```yaml [Standalone alt.] -anonymizer: - yaml: './db_tools.anonymization.yaml' +```ini [Environment] +DBTOOLS_STORAGE_ROOT_DIR="./var/db_tools" ``` ::: @@ -381,16 +464,23 @@ None yet, all options can be used in the standalone console version as well. ## Standalone specific options -### `workdir` +### `anonymization.tables` -Default path in which all relative file system path found in the same config -path will be relative to. +You can write anonymization configuration directly in the configuration file when +using the standalone mode. This prevent configuration file profileration. -If none set, directory in which the configuration file is will be used instead. +Configuration file can be dumped from the Symfony bundle, then used with the +standalone connection. :::code-group ```yaml [Standalone] -workdir: /some/project/path/config +anonymization: + tables: + connection_one: + table_name: + column_name: + anonymizer: anonymizer_name + # ... other options... ``` ::: @@ -400,36 +490,53 @@ All reachable connection list, with their an URL connection string. In standalone mode, connections are handled by `makinacorpus/query-builder`. +:::code-group ```yaml [Standalone] connections: connection_one: "pgsql://username:password@hostname:port?version=16.0&other_option=..." connection_two: "mysql://username:password@hostname:port?version=8.1&other_option=..." ``` +```ini [Environment] +# Only supports single connection. +DBTOOLS_CONNECTION="pgsql://username:password@hostname:port?version=16.0&other_option=..." +``` +::: + +:::warning +Multiple connections configuration is not possible using environment variables yet. +::: + ### `default_connection` Default connection name when connection is unspecified in the command line. If none set, the first one in list will be used instead. +:::code-group ```yaml [Standalone] default_connection: connection_one ``` -### `anonymization.tables` +```ini [Environment] +# Only supports single connection. +DBTOOLS_DEFAULT_CONNECTION="connection_one" +``` +::: -You can write anonymization configuration directly in the configuration file when -using the standalone mode. This prevent configuration file profileration. +### `workdir` -Configuration file can be dumped from the Symfony bundle, then used with the -standalone connection. +Default path in which all relative file system path found in the same config +path will be relative to. + +If none set, directory in which the configuration file is will be used instead. +:::code-group ```yaml [Standalone] -anonymization: - tables: - connection_one: - table_name: - column_name: - anonymizer: anonymizer_name - # ... other options... +workdir: /some/project/path/config ``` + +```ini [Environment] +DBTOOLS_WORKDIR="/some/project/path/config" +``` +::: diff --git a/docs/content/console.md b/docs/content/console.md new file mode 100644 index 00000000..a2e3a2fb --- /dev/null +++ b/docs/content/console.md @@ -0,0 +1,94 @@ +# CLI tool + +DbTools ships a standalone console tool for running it as an standalone application. + +If installed via composer, simply run: + +```sh +./vendor/bin/db-tools +``` + +This CLI tool runs outside of the framework and application context, therefore requires +that you provide a dedicated configuration file for it. + +:::info +A script that generates a working PHAR file is provided in the repository and can be +used to create a self-contained PHAR archive for running in a docker container, on +a baremetal with PHP installed, etc... + +Future plans are to provide an official PHAR archive for each stable release. +::: + +## Configuration + +### Configuration file + +The configuration file is a `YAML` file whose available options are strictly identical +to the Symfony bundle configuration with a few additional parameters dedicated to the +standalone application. + +:::tip +Please refer to the [configuration reference](configuration/reference) for a complete +and detailled configuration options list. +::: + +The most important and required one is the list of available database connections, +using the `connections` configuration option. + +```yaml [Standalone] +connections: + connection_one: "pgsql://username:password@hostname:port?version=16.0&other_option=..." + connection_two: "mysql://username:password@hostname:port?version=8.1&other_option=..." +``` + +Keys are connection names for idenfying those as a command line option, values are +database URL containing all necessary information and options for connecting. + +### Environment variables + +@todo + +### Anonymizer mapping + +Anonymizer tables and columns cannot be configured via command line options since +they necessitate a more complex configuration representation. In order to use the +standalone CLI for anonymizing, you are required to create a mapping configuration +file. + +File structure is the same as the documented YAML anonymization configuration file +which can be used as-is. + +In order to pass configuration, use the `--anonymizer-config=` option or +`DB_TOOLS_ANONYMIZER_CONFIG=` environment variable, followed by a file relative +or absolute path. + +For example: + +::: code-group +```sh [Option] +./vendor/bin/db-tools --anonymizer-config=my_config.yaml +``` + +```sh [Env] +DB_TOOLS_ANONYMIZER_CONFIG=my_config.yaml ./vendor/bin/db-tools +``` +::: + +### Dumping from Symfony configuration + +:::warning +When using the CLI tool, you are not in the Symfony application context anymore, +which means the CLI tool doesn't know the Symfony database configuration, doctrine +connections or doctrine ORM mapping. +::: + +@todo + +## Backup database + + + +## Restore database + + +## Anonymize diff --git a/src/Bridge/Standalone/Bootstrap.php b/src/Bridge/Standalone/Bootstrap.php index 4f865084..b726c490 100644 --- a/src/Bridge/Standalone/Bootstrap.php +++ b/src/Bridge/Standalone/Bootstrap.php @@ -198,13 +198,13 @@ public static function bootstrap(array $config = [], array $configFiles = [], ?L $anonymizerRegistry = self::createAnonymizeRegistry($config); $anonymizatorFactory = new AnonymizatorFactory($databaseSessionRegistry, $anonymizerRegistry, $logger); - $backupperBinaries = $config['backupper_binaries']; - $backupperExcludedTables = $config['excluded_tables'] ?? []; - $backupperOptions = $config['backupper_options']; + $backupperBinaries = $config['backup_binaries']; + $backupperExcludedTables = $config['backup_excluded_tables'] ?? []; + $backupperOptions = $config['backup_options']; $backupperFactory = new BackupperFactory($databaseSessionRegistry, $backupperBinaries, $backupperOptions, $backupperExcludedTables, $logger); - $restorerBinaries = $config['restorer_binaries']; - $restorerOptions = $config['restorer_options']; + $restorerBinaries = $config['restore_binaries']; + $restorerOptions = $config['restore_options']; $restorerFactory = new RestorerFactory($databaseSessionRegistry, $restorerBinaries, $restorerOptions, $logger); $statsProviderFactory = new StatsProviderFactory($databaseSessionRegistry); @@ -307,7 +307,8 @@ private static function configParse(array $config, array $files, LoggerInterface $configs[] = self::configParseFile($filename); } $configs[] = $config; - $configs[] = self::configGetEnv(); + + $config = self::configGetEnv($config); // Use symfony/config and our bundle configuration, which allows us // to use it fully for validation and merge. @@ -378,15 +379,58 @@ private static function configParseFile(string $filename): array /** * Get config variables from environment variables. */ - private static function configGetEnv(): array + private static function configGetEnv(array $config): array { - $config = []; - - // @todo read env variables, validate each, override $config + if (!isset($config['backup_binaries']['mariadb'])) { + $config['backup_binaries']['mariadb'] = self::getEnv('DBTOOLS_BACKUP_BINARY_MARIADB') ?? 'mariadb-dump'; + } + if (!isset($config['backup_binaries']['mysql'])) { + $config['backup_binaries']['mysql'] = self::getEnv('DBTOOLS_BACKUP_BINARY_MYSQL') ?? 'mysqldump'; + } + if (!isset($config['backup_binaries']['postgresql'])) { + $config['backup_binaries']['postgresql'] = self::getEnv('DBTOOLS_BACKUP_BINARY_POSTGRESQL') ?? 'pg_dump'; + } + if (!isset($config['backup_binaries']['sqlite'])) { + $config['backup_binaries']['sqlite'] = self::getEnv('DBTOOLS_BACKUP_BINARY_SQLITE') ?? 'sqlite3'; + } + if (!isset($config['backup_expiration_age'])) { + $config['backup_expiration_age'] = self::getEnv('DBTOOLS_BACKUP_EXPIRATION_AGE') ?? '3 months ago'; + } + if (!isset($config['backup_timeout'])) { + $config['backup_timeout'] = self::getEnv('DBTOOLS_BACKUP_TIMEOUT') ?? '600'; + } + if (!isset($config['restore_binaries']['mariadb'])) { + $config['restore_binaries']['mariadb'] = self::getEnv('DBTOOLS_RESTORE_BINARY_MARIADB') ?? 'mariadb'; + } + if (!isset($config['restore_binaries']['mysql'])) { + $config['restore_binaries']['mysql'] = self::getEnv('DBTOOLS_RESTORE_BINARY_MYSQL') ?? 'mysql'; + } + if (!isset($config['restore_binaries']['postgresql'])) { + $config['restore_binaries']['postgresql'] = self::getEnv('DBTOOLS_RESTORE_BINARY_POSTGRESQL') ?? 'pg_restore'; + } + if (!isset($config['restore_binaries']['sqlite'])) { + $config['restore_binaries']['sqlite'] = self::getEnv('DBTOOLS_RESTORE_BINARY_SQLITE') ?? 'sqlite3'; + } + if (!isset($config['restore_timeout'])) { + $config['restore_timeout'] = self::getEnv('DBTOOLS_RESTORE_TIMEOUT') ?? '1800'; + } + if (!isset($config['storage']['filename_strategy'])) { + $config['storage']['filename_strategy'] = self::getEnv('DBTOOLS_STORAGE_FILENAME_STRATEGY') ?? 'datetime'; + } + if (!isset($config['storage']['root_dir'])) { + $config['storage']['root_dir'] = self::getEnv('DBTOOLS_STORAGE_ROOT_DIR') ?? './var/db_tools'; + } return $config; } + private static function getEnv(string $name): string|null + { + $value = \getenv($name); + + return (!$value && $value !== '0') ? null : (string) $value; + } + /** * Create anonymizer registry and register custom code and additional packs. */ diff --git a/src/Bridge/Standalone/StandaloneConfiguration.php b/src/Bridge/Standalone/StandaloneConfiguration.php index cc07cd0c..87f6dc3e 100644 --- a/src/Bridge/Standalone/StandaloneConfiguration.php +++ b/src/Bridge/Standalone/StandaloneConfiguration.php @@ -9,12 +9,6 @@ class StandaloneConfiguration extends DbToolsConfiguration { - #[\Override] - protected function getDefaultStoragePath(): ?string - { - return './var/db_tools'; - } - #[\Override] public function getConfigTreeBuilder(): TreeBuilder { diff --git a/src/Bridge/Symfony/DependencyInjection/DbToolsConfiguration.php b/src/Bridge/Symfony/DependencyInjection/DbToolsConfiguration.php index e47d0e3e..54942618 100644 --- a/src/Bridge/Symfony/DependencyInjection/DbToolsConfiguration.php +++ b/src/Bridge/Symfony/DependencyInjection/DbToolsConfiguration.php @@ -9,14 +9,6 @@ class DbToolsConfiguration implements ConfigurationInterface { - /** - * Default storage path cannot use variable when standalone. - */ - protected function getDefaultStoragePath(): ?string - { - return '%kernel.project_dir%/var/db_tools'; - } - #[\Override] public function getConfigTreeBuilder(): TreeBuilder { @@ -49,57 +41,73 @@ public function getConfigTreeBuilder(): TreeBuilder $treeBuilder ->getRootNode() ->children() - ->scalarNode('storage_directory') - ->setDeprecated('makinacorpus/db-tools-bundle', '1.0.1', 'Please use "db_tools.storage.root_dir" instead.') - ->end() ->arrayNode('storage') ->addDefaultsIfNotSet() ->children() - ->scalarNode('root_dir')->defaultValue($this->getDefaultStoragePath())->end() + ->scalarNode('root_dir')->defaultNull()->end() ->arrayNode('filename_strategy') + ->beforeNormalization()->ifString()->then(function ($v) { return ['default' => $v]; })->end() ->useAttributeAsKey('connection') ->scalarPrototype()->end() ->end() ->end() ->end() - ->scalarNode('backup_expiration_age')->defaultValue('3 months ago')->end() + ->scalarNode('backup_expiration_age')->end() ->scalarNode('backup_timeout') ->beforeNormalization()->always($intervalToInt)->end() - ->defaultValue(600) ->end() ->scalarNode('restore_timeout') ->beforeNormalization()->always($intervalToInt)->end() - ->defaultValue(1800) ->end() + // @todo Remove in 3.x ->arrayNode('excluded_tables') + ->setDeprecated('makinacorpus/db-tools-bundle', '2.0.0', 'Please use "db_tools.backup_excluded_tables" instead.') + ->beforeNormalization()->always(function ($v) { return \array_is_list($v) ? ['default' => $v] : $v; })->end() + ->useAttributeAsKey('connection') + ->arrayPrototype() + ->scalarPrototype()->end() + ->end() + ->end() + ->arrayNode('backup_excluded_tables') + ->beforeNormalization()->always(function ($v) { return \array_is_list($v) ? ['default' => $v] : $v; })->end() ->useAttributeAsKey('connection') ->arrayPrototype() ->scalarPrototype()->end() ->end() ->end() + // @todo Remove in 3.x ->arrayNode('backupper_binaries') - ->defaultValue([ - 'mariadb' => 'mariadb-dump', - 'mysql' => 'mysqldump', - 'postgresql' => 'pg_dump', - 'sqlite' => 'sqlite3', - ]) + ->setDeprecated('makinacorpus/db-tools-bundle', '2.0.0', 'Please use "db_tools.backup_binaries" instead.') ->scalarPrototype()->end() ->end() + ->arrayNode('backup_binaries') + ->scalarPrototype()->end() + ->end() + // @todo Remove in 3.x ->arrayNode('restorer_binaries') - ->defaultValue([ - 'mariadb' => 'mariadb', - 'mysql' => 'mysql', - 'postgresql' => 'pg_restore', - 'sqlite' => 'sqlite3', - ]) + ->setDeprecated('makinacorpus/db-tools-bundle', '2.0.0', 'Please use "db_tools.restore_binaries" instead.') ->scalarPrototype()->end() ->end() + ->arrayNode('restore_binaries') + ->scalarPrototype()->end() + ->end() + // @todo Remove in 3.x ->arrayNode('backupper_options') + ->setDeprecated('makinacorpus/db-tools-bundle', '2.0.0', 'Please use "db_tools.backup_options" instead.') + ->useAttributeAsKey('connection') + ->scalarPrototype()->end() + ->end() + ->arrayNode('backup_options') ->useAttributeAsKey('connection') ->scalarPrototype()->end() ->end() + // @todo Remove in 3.x ->arrayNode('restorer_options') + ->setDeprecated('makinacorpus/db-tools-bundle', '2.0.0', 'Please use "db_tools.restore_options" instead.') + ->useAttributeAsKey('connection') + ->scalarPrototype()->end() + ->end() + ->arrayNode('restore_options') ->useAttributeAsKey('connection') ->scalarPrototype()->end() ->end() diff --git a/src/Bridge/Symfony/DependencyInjection/DbToolsExtension.php b/src/Bridge/Symfony/DependencyInjection/DbToolsExtension.php index d13b4eaf..6e716cc6 100644 --- a/src/Bridge/Symfony/DependencyInjection/DbToolsExtension.php +++ b/src/Bridge/Symfony/DependencyInjection/DbToolsExtension.php @@ -25,24 +25,59 @@ public function load(array $configs, ContainerBuilder $container): void $loader = new YamlFileLoader($container, new FileLocator(\dirname(__DIR__).'/Resources/config')); $loader->load('services.yaml'); - if (isset($config['storage_directory'])) { - \trigger_deprecation('makinacorpus/db-tools-bundle', '1.0.1', '"db_tools.storage_directory" configuration option is deprecated and renamed "db_tools.storage.root_dir"'); - $container->setParameter('db_tools.storage.root_dir', $config['storage_directory']); - } else { + // @todo Remove in 3.x. + $deprecationMap = [ + 'backupper_binaries' => 'backup_binaries', + 'backupper_options' => 'backup_options', + 'excluded_tables' => 'backup_excluded_tables', + 'restorer_binaries' => 'restore_binaries', + 'restorer_options' => 'restore_options', + ]; + foreach ($deprecationMap as $legacyName => $newName) { + if (!empty($config[$legacyName])) { + \trigger_deprecation('makinacorpus/db-tools-bundle', '2.0.0', '"db_tools.%s" configuration option is deprecated and renamed "db_tools.%s"', $legacyName, $newName); + $config[$newName] = $config[$legacyName]; + } + unset($config[$legacyName]); + } + + // Those parameters default values are environment variable defaults + // as seen in ../Resources/config/services.yaml. We override parameter + // values only if the user explicitely defined it; otherwise it would + // prevent environment variables completely. + if (isset($config['backup_expiration_age'])) { + $container->setParameter('db_tools.backup_expiration_age', $config['backup_expiration_age']); + } + if (isset($config['backup_options'])) { // @todo Not in env. variables yet. + $container->setParameter('db_tools.backup_options', $config['backup_options']); + } + if (isset($config['backup_timeout'])) { + $container->setParameter('db_tools.backup_timeout', $config['backup_timeout']); + } + if (isset($config['restore_options'])) { // @todo Not in env. variables yet. + $container->setParameter('db_tools.restore_options', $config['restore_options']); + } + if (isset($config['restore_timeout'])) { + $container->setParameter('db_tools.restore_timeout', $config['restore_timeout']); + } + if (isset($config['storage']['root_dir'])) { $container->setParameter('db_tools.storage.root_dir', $config['storage']['root_dir']); } - // Backupper - $container->setParameter('db_tools.backupper.binaries', $config['backupper_binaries']); - $container->setParameter('db_tools.backupper.options', $config['backupper_options']); - $container->setParameter('db_tools.backup_expiration_age', $config['backup_expiration_age']); - $container->setParameter('db_tools.excluded_tables', $config['excluded_tables'] ?? []); - $container->setParameter('db_tools.backup_timeout', $config['backup_timeout']); + // Special treatment for binaries, because the backupper and restorer + // services await for an array of values. + foreach (['backup_binaries', 'restore_binaries'] as $prefix) { + foreach (['mariadb', 'mysql', 'postgresql', 'sqlite'] as $vendor) { + if (isset($config[$prefix][$vendor])) { + $container->setParameter('db_tools.' . $prefix . '.' . $vendor, $config[$prefix][$vendor]); + } + } + } - // Restorer - $container->setParameter('db_tools.restorer.binaries', $config['restorer_binaries']); - $container->setParameter('db_tools.restorer.options', $config['restorer_options']); - $container->setParameter('db_tools.restore_timeout', $config['restore_timeout']); + // Those parameters are NOT in environment variables. + // Excluded tables is dependent on the application schema and not + // a runtime parameter, its place is not in environment variables. + $container->setParameter('db_tools.backup_excluded_tables', $config['backup_excluded_tables'] ?? []); // Validate user-given anonymizer paths. $anonymizerPaths = $config['anonymizer_paths']; diff --git a/src/Bridge/Symfony/Resources/config/services.yaml b/src/Bridge/Symfony/Resources/config/services.yaml index c1e5c1e4..2881876b 100644 --- a/src/Bridge/Symfony/Resources/config/services.yaml +++ b/src/Bridge/Symfony/Resources/config/services.yaml @@ -1,3 +1,42 @@ +parameters: + # Environment variables with default option values. All those environment + # variables will be used per default, if defined, but will be ignored when + # you override the related parameter in the "config/packages/db_tools.yaml" + # file. + env(DBTOOLS_BACKUP_BINARY_MARIADB): 'mariadb-dump' + env(DBTOOLS_BACKUP_BINARY_MYSQL): 'mysqldump' + env(DBTOOLS_BACKUP_BINARY_POSTGRESQL): 'pg_dump' + env(DBTOOLS_BACKUP_BINARY_SQLITE): 'sqlite3' + env(DBTOOLS_BACKUP_EXPIRATION_AGE): '3 months ago' + #env(DBTOOLS_BACKUP_OPTIONS): ~ + env(DBTOOLS_BACKUP_TIMEOUT): '600' + env(DBTOOLS_RESTORE_BINARY_MARIADB): 'mariadb' + env(DBTOOLS_RESTORE_BINARY_MYSQL): 'mysql' + env(DBTOOLS_RESTORE_BINARY_POSTGRESQL): 'pg_restore' + env(DBTOOLS_RESTORE_BINARY_SQLITE): 'sqlite3' + #env(DBTOOLS_RESTORE_OPTIONS): ~ + env(DBTOOLS_RESTORE_TIMEOUT): '1800' + env(DBTOOLS_STORAGE_FILENAME_STRATEGY): datetime + env(DBTOOLS_STORAGE_ROOT_DIR): "%kernel.project_dir%/var/db_tools" + + # Container parameters that maps per default on the environment variables + # and will be injected to services. Those parameter values will be overriden + # by user given values from the "config/packages/db_tools.yaml" file in the + # MakinaCorpus\DbToolsBundle\Bridge\Symfony\DependencyInjection\DbToolsExtension + # class. + db_tools.backup_binaries.mariadb: "%env(resolve:DBTOOLS_BACKUP_BINARY_MARIADB)%" + db_tools.backup_binaries.mysql: "%env(resolve:DBTOOLS_BACKUP_BINARY_MYSQL)%" + db_tools.backup_binaries.postgresql: "%env(resolve:DBTOOLS_BACKUP_BINARY_POSTGRESQL)%" + db_tools.backup_binaries.sqlite: "%env(resolve:DBTOOLS_BACKUP_BINARY_SQLITE)%" + db_tools.backup_expiration_age: "%env(DBTOOLS_BACKUP_EXPIRATION_AGE)%" + db_tools.backup_timeout: "%env(int:DBTOOLS_BACKUP_TIMEOUT)%" + db_tools.restore_binaries.mariadb: "%env(resolve:DBTOOLS_RESTORE_BINARY_MARIADB)%" + db_tools.restore_binaries.mysql: "%env(resolve:DBTOOLS_RESTORE_BINARY_MYSQL)%" + db_tools.restore_binaries.postgresql: "%env(resolve:DBTOOLS_RESTORE_BINARY_POSTGRESQL)%" + db_tools.restore_binaries.sqlite: "%env(resolve:DBTOOLS_RESTORE_BINARY_SQLITE)%" + db_tools.restore_timeout: '%env(int:DBTOOLS_RESTORE_TIMEOUT)%' + db_tools.storage.root_dir: '%env(resolve:DBTOOLS_STORAGE_ROOT_DIR)%' + services: # Commands db_tools.command.anonymization.run: @@ -72,9 +111,13 @@ services: class: MakinaCorpus\DbToolsBundle\Backupper\BackupperFactory arguments: - '@db_tools.database_session.registry' - - '%db_tools.backupper.binaries%' - - '%db_tools.backupper.options%' - - '%db_tools.excluded_tables%' + - + mariadb: '%db_tools.backup_binaries.mariadb%' + mysql: '%db_tools.backup_binaries.mysql%' + postgresql: '%db_tools.backup_binaries.postgresql%' + sqlite: '%db_tools.backup_binaries.sqlite%' + - '%db_tools.backup_options%' + - '%db_tools.backup_excluded_tables%' - '@logger' tags: [{ name: 'monolog.logger', channel: 'db_tools_backup' }] @@ -83,8 +126,12 @@ services: class: MakinaCorpus\DbToolsBundle\Restorer\RestorerFactory arguments: - '@db_tools.database_session.registry' - - '%db_tools.restorer.binaries%' - - '%db_tools.restorer.options%' + - + mariadb: '%db_tools.restore_binaries.mariadb%' + mysql: '%db_tools.restore_binaries.mysql%' + postgresql: '%db_tools.restore_binaries.postgresql%' + sqlite: '%db_tools.restore_binaries.sqlite%' + - '%db_tools.restore_options%' - '@logger' tags: [{ name: monolog.logger, channel: db_tools_restoration }] diff --git a/src/Helper/Cli/Bootstrap.php b/src/Helper/Cli/Bootstrap.php deleted file mode 100644 index c6562ed5..00000000 --- a/src/Helper/Cli/Bootstrap.php +++ /dev/null @@ -1,466 +0,0 @@ -run(); - } - - /** - * Create Symfony console application. - */ - public static function createApplication(): Application - { - // @todo Test in PHAR context. - if (\class_exists(InstalledVersions::class)) { - $version = InstalledVersions::getVersion('makinacorpus/db-tools-bundle'); - } - $version ??= 'cli'; - \assert($version !== null); - - $application = new Application('Db Tools', $version); - $application->setCatchExceptions(true); - $application->setDefaultCommand('list'); - - $definition = $application->getDefinition(); - $definition->addOption(new InputOption('config', null, InputOption::VALUE_REQUIRED | InputOption::VALUE_IS_ARRAY, 'Configuration files', null)); - $definition->addOption(new InputOption('env', null, InputOption::VALUE_REQUIRED, 'Environment', 'dev')); - - // Hack, we need it output to have the same configuration as the application. - $input = new ArgvInput(); - $output = new ConsoleOutput(); - (\Closure::bind(fn () => $application->configureIO($input, $output), null, Application::class))(); - - // We need to parse a few arguments prior running the console - // application in order to setup commands. This is hackish but - // should work. - $config = $configFiles = []; - if ($input->hasOption('config')) { - foreach ((array) $input->getOption('config') as $filename) { - $configFiles[] = $filename; - } - } - - $commands = [ - 'anonymization:clean' => [ - fn (Context $context) => new CleanCommand( - anonymizatorFactory: $context->anonymizatorFactory, - defaultConnectionName: $context->databaseSessionRegistry->getDefaultConnectionName(), - ), - 'Clean DbTools left-over temporary tables', - ['clean'], - ], - 'anonymization:config-dump' => [ - fn (Context $context) => new ConfigDumpCommand( - anonymizatorFactory: $context->anonymizatorFactory, - ), - 'Dump anonymization configuration', - ['config-dump'], - ], - 'anonymization:list' => [ - fn (Context $context) => new AnonymizerListCommand( - anonymizerRegistry: $context->anonymizerRegistry, - ), - 'List all available anonymizers', - [], - ], - 'anonymization:run' => [ - fn (Context $context) => new AnonymizeCommand( - defaultConnectionName: $context->databaseSessionRegistry->getDefaultConnectionName(), - restorerFactory: $context->restorerFactory, - backupperFactory: $context->backupperFactory, - anonymizatorFactory: $context->anonymizatorFactory, - storage: $context->storage, - ), - 'Anonymize given backup file or the local database', - ['anonymize'], - ], - 'database:backup' => [ - fn (Context $context) => new BackupCommand( - defaultConnectionName: $context->databaseSessionRegistry->getDefaultConnectionName(), - backupperFactory: $context->backupperFactory, - storage: $context->storage, - ), - 'Backup database', - ['backup'], - ], - 'database:check' => [ - fn (Context $context) => new CheckCommand( - defaultConnectionName: $context->databaseSessionRegistry->getDefaultConnectionName(), - backupperFactory: $context->backupperFactory, - restorerFactory: $context->restorerFactory, - ), - 'Check backup and restore binaries', - ['check'], - ], - 'database:restore' => [ - fn (Context $context) => new RestoreCommand( - defaultConnectionName: $context->databaseSessionRegistry->getDefaultConnectionName(), - restorerFactory: $context->restorerFactory, - storage: $context->storage, - ), - 'Restore database.', - ['restore'], - ], - 'database:stats' => [ - fn (Context $context) => new StatsCommand( - defaultConnectionName: $context->databaseSessionRegistry->getDefaultConnectionName(), - statsProviderFactory: $context->statsProviderFactory, - ), - 'Give some database statistics', - ['stats'], - ], - ]; - - $initializer = static fn (): Context => self::bootstrap($config, $configFiles, new ConsoleLogger($output)); - - // All commands are wrapped into LazyCommand instances, we do not - // really care about performances here, we have really few commands - // and it's OK to initialiaze them all, but we need to change their - // name to shorten them. - foreach ($commands as $name => $data) { - list($callback, $description, $aliases) = $data; - $application->add( - new LazyCommand( - name: $name, - aliases: $aliases, - description: $description, - isHidden: false, - commandFactory: fn () => $callback($initializer()), - ), - ); - } - - return $application; - } - - /** - * Bootstrap components as a standalone application. - * - * @param array $config - * Configuration parsed from application bootstrap using CLI options. - * This configuration must match the Symfony configuration file without - * the "db_tools" root level. - * @param array $configFiles - * Additional configuration files to parse. - */ - public static function bootstrap(array $config = [], array $configFiles = [], ?LoggerInterface $logger = null): Context - { - $logger ?? new NullLogger(); - $config = self::configParse($config, $configFiles, $logger); - - $databaseSessionRegistry = self::createDatabaseSessionRegistry($config); - - $anonymizerRegistry = self::createAnonymizeRegistry($config); - $anonymizatorFactory = new AnonymizatorFactory($databaseSessionRegistry, $anonymizerRegistry, $logger); - - $backupperBinaries = $config['backupper_binaries']; - $backupperExcludedTables = $config['excluded_tables'] ?? []; - $backupperOptions = $config['backupper_options']; - $backupperFactory = new BackupperFactory($databaseSessionRegistry, $backupperBinaries, $backupperOptions, $backupperExcludedTables, $logger); - - $restorerBinaries = $config['restorer_binaries']; - $restorerOptions = $config['restorer_options']; - $restorerFactory = new RestorerFactory($databaseSessionRegistry, $restorerBinaries, $restorerOptions, $logger); - - $statsProviderFactory = new StatsProviderFactory($databaseSessionRegistry); - $storage = self::createStorage($config, $logger); - - return new Context( - anonymizatorFactory: $anonymizatorFactory, - anonymizerRegistry: $anonymizerRegistry, - backupperFactory: $backupperFactory, - databaseSessionRegistry: $databaseSessionRegistry, - logger: $logger, - restorerFactory: $restorerFactory, - statsProviderFactory: $statsProviderFactory, - storage: $storage, - ); - } - - /** - * Gets the application root dir (path of the project's composer file). - */ - private static function getProjectDir(LoggerInterface $logger): ?string - { - // 4 level of \dirname() gets us in this project parent folder. - $candidates = [\getcwd(), \dirname(__DIR__, 4)]; - - foreach ($candidates as $candidate) { - $dir = $candidate; - while ($dir) { - if (\is_file($dir.'/composer.json')) { - $logger->notice('Project root directory found: {dir}', ['dir' => $dir]); - - return $dir; - } - $logger->debug('Not found project directory: {dir}', ['dir' => $dir]); - $dir = \dirname($dir); - } - } - return null; - } - - /** - * Parse configuration files, and environment provided configuration. - * - * @param array $config - * Overriding configuration from user input. It will overide configuration - * from given files. - * @param array $files - * Configuration files, in override order in case of conflict. - * - * @return array - * Merged proper configuration. - */ - private static function configParse(array $config, array $files, LoggerInterface $logger): array - { - $configFileNames = ['db_tools.yaml', 'db_tools.yml', 'db_tools.config.yaml', 'db_tools.config.yml']; - $projectRoot = self::getProjectDir($logger); - $workdir = $config['workdir'] ?? $projectRoot ?? \getcwd(); - - // When no configuration file given, attempt to find one. - // @todo Should stop at first when found. - if (empty($files)) { - $candidates = []; - if ($projectRoot) { - foreach ($configFileNames as $filename) { - $candidates[] = self::pathConcat($projectRoot, $filename); - } - } - if ($projectRoot !== $workdir) { - foreach ($configFileNames as $filename) { - $candidates[] = self::pathConcat($workdir, $filename); - } - } - // Will not work under Windows (and that's OK). - if ($homedir = \getenv("HOME")) { - // @todo .config folder is configurable with XDG portals? - $candidates[] = self::pathConcat($homedir, '/.config/db_tools/config.yaml'); - $candidates[] = self::pathConcat($homedir, '/.config/db_tools/config.yml'); - // As dot files. - foreach ($configFileNames as $filename) { - $candidates[] = self::pathConcat($homedir, '.' . $filename); - } - } - - foreach ($candidates as $filename) { - if (\file_exists($filename)) { - if (\is_readable($filename)) { - $logger->notice("Found configuration file: {file}", ['file' => $filename]); - $files[] = $filename; - } else { - $logger->warning("Configuration file could not be read: {file}", ['file' => $filename]); - } - } else { - $logger->debug("Configuration file does not exist: {file}", ['file' => $filename]); - } - } - } - - $configs = []; - foreach ($files as $filename) { - $configs[] = self::configParseFile($filename); - } - $configs[] = $config; - $configs[] = self::configGetEnv(); - - // Use symfony/config and our bundle configuration, which allows us - // to use it fully for validation and merge. - $configuration = new StandaloneConfiguration(); - $processor = new Processor(); - - $config = $processor->processConfiguration($configuration, $configs); - - // Set a base directory for file and backup lookup. - $config['workdir'] ?? $workdir; - - return $config; - } - - /** - * Parse a single configuration file. - */ - private static function configParseFile(string $filename): array - { - if (!\file_exists($filename)) { - throw new ConfigurationException(\sprintf("%s: file does not exist.", $filename)); - } - if (!\is_readable($filename)) { - throw new ConfigurationException(\sprintf("%s: file cannot be read.", $filename)); - } - - // 0 is not a good index for extension, this fails for false and 0. - if (!($pos = \strrpos($filename, '.'))) { - throw new ConfigurationException(\sprintf("%s: file extension cannot be guessed.", $filename)); - } - $ext = \substr($filename, $pos + 1); - - $config = match ($ext) { - 'json' => \json_decode(\file_get_contents($filename), true, 512, \JSON_THROW_ON_ERROR), - 'yml', 'yaml' => Yaml::parseFile($filename), - default => throw new ConfigurationException(\sprintf("%s: file extension '%s' is unsupported.", $filename, $ext)), - }; - - // Resolve all known filenames relative to this file. - // @todo Warning, this code will only work on UNIX-like filesystems. - $workdir = \rtrim($config['workdir'] ?? \dirname($filename), '/'); - - // Storage root directory. - if ($path = ($config['storage']['root_dir'] ?? null)) { - $config['storage']['root_dir'] = self::pathAbs($workdir, $path); - } - - // YAML anonymizer file paths. - $yaml = $config['anonymization']['yaml'] ?? null; - if (isset($yaml)) { - if (\is_array($yaml)) { - foreach ($yaml as $name => $path) { - $config['anonymization']['yaml'][$name] = self::pathAbs($workdir, $path); - } - } else { - $config['anonymization']['yaml'] = self::pathAbs($workdir, $yaml); - } - } - - // Custom anonymizer paths. - foreach (($config['anonymizer_paths'] ?? []) as $index => $path) { - $config['anonymizer_paths'][$index] = self::pathAbs($workdir, $path); - } - - return $config; - } - - /** - * Get config variables from environment variables. - */ - private static function configGetEnv(): array - { - $config = []; - - // @todo read env variables, validate each, override $config - - return $config; - } - - /** - * Create anonymizer registry and register custom code and additional packs. - */ - private static function createAnonymizeRegistry(array $config): AnonymizerRegistry - { - $projectDir = null; - $paths = []; - - // @todo find a way to register packs when not in a composer project - - return new AnonymizerRegistry($projectDir, $paths); - } - - /** - * Create database session registry from config-given connections. - */ - private static function createDatabaseSessionRegistry(array $config): DatabaseSessionRegistry - { - if (empty($config['connections'])) { - throw new ConfigurationError("No database connection found, this means that either you forgot it into your configuration file, or no configuration files were found. Please run using the -vvv switch for more information."); - } - - return new StandaloneDatabaseSessionRegistry($config['connections'], $config['default_connection']); - } - - /** - * Create storage. - */ - private static function createStorage(array $config, LoggerInterface $logger): Storage - { - $rootdir = $config['storage']['root_dir'] ?? $config['workdir']; - - if (!\is_dir($rootdir)) { - if (\file_exists($rootdir)) { - throw new ConfigurationException(\sprintf("Storage root folder is a regular file instead of a directory: %s", $rootdir)); - } - - $logger->notice("Storage root folder does not exists: {dir}", ['dir' => $rootdir]); - } else { - $logger->notice("Found storage root folder: {dir}", ['dir' => $rootdir]); - } - - return new Storage($config['storage']['root_dir'], $config['backup_expiration_age']); - } - - /** - * Concat and make absolute using given root if relative. - */ - private static function pathAbs(string $root, string ...$pieces): string - { - $path = self::pathConcat(...$pieces); - if (\str_starts_with($path, '/')) { - return $path; - } - return self::pathConcat($root, $path); - } - - /** - * Concat all path segment while cleaning a bit given input. - */ - private static function pathConcat(string ...$pieces): string - { - $first = true; - foreach ($pieces as $index => $piece) { - if ($first) { - $first = false; - $pieces[$index] = \rtrim($piece, '/\\'); - } else { - if (\str_starts_with($piece, './')) { - $piece = \substr($piece, 2); - } - $pieces[$index] = \trim($piece, '/\\'); - } - } - return \implode(DIRECTORY_SEPARATOR, $pieces); - } -} diff --git a/src/Helper/Cli/Context.php b/src/Helper/Cli/Context.php deleted file mode 100644 index c0276060..00000000 --- a/src/Helper/Cli/Context.php +++ /dev/null @@ -1,31 +0,0 @@ -getRootNode() - ->children() - ->scalarNode('workdir') - ->info('Directory path all other files will be relative to, if none providen then the configuration file directory will be used instead.') - ->defaultNull() - ->end() - ->arrayNode('connections') - ->normalizeKeys(true) - ->scalarPrototype() - ->info('Database connection DSN/URL.') - ->end() - ->end() - ->scalarNode('default_connection') - ->info('Default connection name. If none providen, first one is used instead.') - ->defaultNull() - ->end() - ->arrayNode('anonymization') - ->children() - ->arrayNode('tables') - ->beforeNormalization()->ifString()->then(function ($v) { return ['default' => $v]; })->end() - ->variablePrototype() - ->info('Keys are table names, values are arrays whose keys are column names and values are anonymizer configurations.') - ->end() - ->end() - ->end() - ->end() - ->end() - ->end() - ; - - return $treeBuilder; - } -} diff --git a/src/Storage/Storage.php b/src/Storage/Storage.php index 2b8c57d8..85e5ef24 100644 --- a/src/Storage/Storage.php +++ b/src/Storage/Storage.php @@ -81,6 +81,6 @@ public function getStoragePath(): string protected function getFilenameStrategy(string $connectionName): FilenameStrategyInterface { - return $this->filenameStrategies[$connectionName] ?? new DefaultFilenameStrategy(); + return $this->filenameStrategies[$connectionName] ?? $this->filenameStrategies['default'] ?? new DefaultFilenameStrategy(); } } diff --git a/tests/Resources/config/packages/db_tools_alt1.yaml b/tests/Resources/config/packages/db_tools_alt1.yaml index 512bcec2..7514ddac 100644 --- a/tests/Resources/config/packages/db_tools_alt1.yaml +++ b/tests/Resources/config/packages/db_tools_alt1.yaml @@ -1,25 +1,26 @@ db_tools: - storage_directory: '%kernel.project_dir%/var/backup' + storage: + root_dir: '%kernel.project_dir%/var/backup' backup_expiration_age: '6 months ago' backup_timeout: 1200 restore_timeout: 2400 - excluded_tables: + backup_excluded_tables: default: ['table1', 'table2'] - backupper_binaries: + backup_binaries: mariadb: '/usr/bin/mariadb-dump' mysql: '/usr/bin/mysqldump' postgresql: '/usr/bin/pg_dump' sqlite: '/usr/bin/sqlite3' - restorer_binaries: + restore_binaries: mariadb: '/usr/bin/mariadb' mysql: '/usr/bin/mysql' postgresql: '/usr/bin/pg_restore' sqlite: '/usr/bin/sqlite3' - backupper_options: + backup_options: default: '--opt1 val1 -x -y -z --opt2 val2' - restorer_options: + restore_options: default: '-abc -x val1 -y val2' anonymizer_paths: diff --git a/tests/Resources/config/packages/db_tools_alt2.yaml b/tests/Resources/config/packages/db_tools_alt2.yaml index 1d48166a..095ec339 100644 --- a/tests/Resources/config/packages/db_tools_alt2.yaml +++ b/tests/Resources/config/packages/db_tools_alt2.yaml @@ -6,25 +6,25 @@ db_tools: backup_expiration_age: '6 months ago' backup_timeout: 1800 restore_timeout: 3200 - excluded_tables: + backup_excluded_tables: connection_two: ['table1', 'table2'] - backupper_binaries: + backup_binaries: mariadb: '/usr/bin/mariadb-dump' mysql: '/usr/bin/mysqldump' postgresql: '/usr/bin/pg_dump' sqlite: '/usr/bin/sqlite3' - restorer_binaries: + restore_binaries: mariadb: '/usr/bin/mariadb' mysql: '/usr/bin/mysql' postgresql: '/usr/bin/pg_restore' sqlite: '/usr/bin/sqlite3' - backupper_options: + backup_options: connection_one: '--opt1 val1 -x -y -z --opt2 val2' # Let's say we have no options for connection_two. #connection_two: '' - restorer_options: + restore_options: connection_one: '-abc -x val1 -y val2' connection_two: '-a "Value 1" -bc -d val2 --end' diff --git a/tests/Unit/Bridge/Symfony/DependencyInjection/DbToolsConfigurationTest.php b/tests/Unit/Bridge/Symfony/DependencyInjection/DbToolsConfigurationTest.php index 2498f179..7c607828 100644 --- a/tests/Unit/Bridge/Symfony/DependencyInjection/DbToolsConfigurationTest.php +++ b/tests/Unit/Bridge/Symfony/DependencyInjection/DbToolsConfigurationTest.php @@ -22,77 +22,88 @@ private function processYamlConfiguration(array|string $dataOrFilename): array ); } + private function deprecatedDefaultValues(): array + { + return [ + // @todo Remove in 3.x + 'backupper_binaries' => [], + 'backupper_options' => [], + 'excluded_tables' => [], + 'restorer_binaries' => [], + 'restorer_options' => [], + ]; + } + public function testConfigurationMinimal(): array { $result = $this->processYamlConfiguration( \dirname(__DIR__, 4) . '/Resources/config/packages/db_tools_min.yaml' ); - self::assertSame( + self::assertEquals( [ - 'storage' => [ - 'root_dir' => '%kernel.project_dir%/var/db_tools', - 'filename_strategy' => [], - ], - 'backup_expiration_age' => '3 months ago', - 'backup_timeout' => 600, - 'restore_timeout' => 1800, - 'excluded_tables' => [], - 'backupper_binaries' => [ + 'anonymizer_paths' => [], + 'backup_binaries' => [ 'mariadb' => 'mariadb-dump', 'mysql' => 'mysqldump', 'postgresql' => 'pg_dump', 'sqlite' => 'sqlite3', ], - 'restorer_binaries' => [ + 'backup_expiration_age' => '3 months ago', + 'backup_excluded_tables' => [], + 'backup_options' => [], + 'backup_timeout' => 600, + 'restore_binaries' => [ 'mariadb' => 'mariadb', 'mysql' => 'mysql', 'postgresql' => 'pg_restore', 'sqlite' => 'sqlite3', ], - 'backupper_options' => [], - 'restorer_options' => [], - 'anonymizer_paths' => [], - ], - $result + 'restore_options' => [], + 'restore_timeout' => 1800, + 'storage' => [ + 'root_dir' => '%kernel.project_dir%/var/db_tools', + 'filename_strategy' => [], + ], + ] + $this->deprecatedDefaultValues(), + $result, ); return $result; } - public function testConfigurationAlternative1(): array + public function testConfigurationAlternative1(): void { $result = $this->processYamlConfiguration( \dirname(__DIR__, 4) . '/Resources/config/packages/db_tools_alt1.yaml' ); - self::assertSame( + self::assertEquals( [ - 'storage_directory' => '%kernel.project_dir%/var/backup', 'backup_expiration_age' => '6 months ago', - 'backup_timeout' => 1200, - 'restore_timeout' => 2400, - 'excluded_tables' => [ + 'backup_excluded_tables' => [ 'default' => ['table1', 'table2'], ], - 'backupper_binaries' => [ + 'backup_binaries' => [ 'mariadb' => '/usr/bin/mariadb-dump', 'mysql' => '/usr/bin/mysqldump', 'postgresql' => '/usr/bin/pg_dump', 'sqlite' => '/usr/bin/sqlite3', ], - 'restorer_binaries' => [ + 'restore_binaries' => [ 'mariadb' => '/usr/bin/mariadb', 'mysql' => '/usr/bin/mysql', 'postgresql' => '/usr/bin/pg_restore', 'sqlite' => '/usr/bin/sqlite3', ], - 'backupper_options' => [ + 'backup_options' => [ 'default' => '--opt1 val1 -x -y -z --opt2 val2', ], - 'restorer_options' => [ + 'backup_timeout' => 1200, + 'restore_options' => [ 'default' => '-abc -x val1 -y val2', ], + 'restore_timeout' => 2400, 'anonymizer_paths' => [ '%kernel.project_dir%/vendor/makinacorpus/db-tools-bundle/src/Anonymizer', '%kernel.project_dir%/src/Anonymization/Anonymizer', @@ -103,23 +114,21 @@ public function testConfigurationAlternative1(): array ], ], 'storage' => [ - 'root_dir' => '%kernel.project_dir%/var/db_tools', + 'root_dir' => '%kernel.project_dir%/var/backup', 'filename_strategy' => [], ], - ], - $result + ] + $this->deprecatedDefaultValues(), + $result, ); - - return $result; } - public function testConfigurationAlternative2(): array + public function testConfigurationAlternative2(): void { $result = $this->processYamlConfiguration( \dirname(__DIR__, 4) . '/Resources/config/packages/db_tools_alt2.yaml' ); - self::assertSame( + self::assertEquals( [ 'storage' => [ 'root_dir' => '%kernel.project_dir%/var/backup', @@ -128,30 +137,30 @@ public function testConfigurationAlternative2(): array ], ], 'backup_expiration_age' => '6 months ago', - 'backup_timeout' => 1800, - 'restore_timeout' => 3200, - 'excluded_tables' => [ + 'backup_excluded_tables' => [ 'connection_two' => ['table1', 'table2'], ], - 'backupper_binaries' => [ + 'backup_binaries' => [ 'mariadb' => '/usr/bin/mariadb-dump', 'mysql' => '/usr/bin/mysqldump', 'postgresql' => '/usr/bin/pg_dump', 'sqlite' => '/usr/bin/sqlite3', ], - 'restorer_binaries' => [ + 'restore_binaries' => [ 'mariadb' => '/usr/bin/mariadb', 'mysql' => '/usr/bin/mysql', 'postgresql' => '/usr/bin/pg_restore', 'sqlite' => '/usr/bin/sqlite3', ], - 'backupper_options' => [ + 'backup_options' => [ 'connection_one' => '--opt1 val1 -x -y -z --opt2 val2', ], - 'restorer_options' => [ + 'backup_timeout' => 1800, + 'restore_options' => [ 'connection_one' => '-abc -x val1 -y val2', 'connection_two' => '-a "Value 1" -bc -d val2 --end', ], + 'restore_timeout' => 3200, 'anonymizer_paths' => [ '%kernel.project_dir%/vendor/makinacorpus/db-tools-bundle/src/Anonymizer', '%kernel.project_dir%/src/Anonymization/Anonymizer', @@ -162,11 +171,53 @@ public function testConfigurationAlternative2(): array 'connection_two' => '%kernel.project_dir%/config/anonymizations/connection_two.yaml', ], ], + ] + $this->deprecatedDefaultValues(), + $result, + ); + } + + public function testConfigurationFilenameStrategyNull(): void + { + $result = $this->processYamlConfiguration([ + 'storage' => [ + 'filename_strategy' => null, ], - $result + ]); + + self::assertEqualsCanonicalizing( + [], + $result['storage']['filename_strategy'], ); + } - return $result; + public function testConfigurationFilenameStrategyString(): void + { + $result = $this->processYamlConfiguration([ + 'storage' => [ + 'filename_strategy' => 'some_strategy', + ], + ]); + + self::assertSame( + ['default' => 'some_strategy'], + $result['storage']['filename_strategy'], + ); + } + + public function testConfigurationFilenameStrategyArray(): void + { + $result = $this->processYamlConfiguration([ + 'storage' => [ + 'filename_strategy' => [ + 'default' => 'some_strategy' + ], + ], + ]); + + self::assertSame( + ['default' => 'some_strategy'], + $result['storage']['filename_strategy'], + ); } public function testConfigurationBackupTimeoutInt(): void