diff --git a/.circleci/config.yml b/.circleci/config.yml index a9e9c5fd93e..61333b54659 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,7 +8,7 @@ branch_filters: &job_filters executors: musicbrainz-tests: docker: - - image: metabrainz/musicbrainz-tests:v-2024-05-23 + - image: metabrainz/musicbrainz-tests:v-2024-05-27 user: root working_directory: /home/musicbrainz/musicbrainz-server diff --git a/admin/InitDb.pl b/admin/InitDb.pl index a553bdabbf3..a8f47188e6e 100755 --- a/admin/InitDb.pl +++ b/admin/InitDb.pl @@ -6,10 +6,12 @@ use English; use FindBin; use lib "$FindBin::Bin/../lib"; +use String::ShellQuote qw( shell_quote ); use DBDefs; use MusicBrainz::Server::Constants qw( @FULL_SCHEMA_LIST ); use MusicBrainz::Server::Replication qw( :replication_type ); +use MusicBrainz::Script::Utils qw( find_files ); use aliased 'MusicBrainz::Server::DatabaseConnectionFactory' => 'Databases'; @@ -23,6 +25,7 @@ my $fCreateDB; my $fInstallExtension; my $fExtensionSchema; +my $initial_sql = 'InsertDefaultRows.sql'; my $tmp_dir; use Getopt::Long; @@ -57,11 +60,15 @@ sub RequireMinimumPostgreSQLVersion sub RunSQLScript { - my ($db, $file, $startmessage, $path) = @_; - $startmessage ||= "Running $file"; + my ($db, $file, $startmessage) = @_; + $startmessage ||= "Running $file ..."; print localtime() . " : $startmessage ($file)\n" unless $fQuiet; - $path //= $sqldir; + my $quoted_file_path = do { + my ($file_path) = find_files($file, $file, $sqldir, + DBDefs->MB_SERVER_ROOT); + shell_quote($file_path); + }; my $opts = $db->shell_args; my $echo = ($fEcho ? '-e' : ''); @@ -83,8 +90,8 @@ sub RunSQLScript $ENV{'PGOPTIONS'} .= ' -c client_min_messages=WARNING'; } - print "$psql $quiet $echo -f $path/$file $opts 2>&1 $stdout |\n" if $fVerbose; - open(PIPE, "$psql $quiet $echo -f $path/$file $opts 2>&1 $stdout |") + print "$psql $quiet $echo -f $quoted_file_path $opts 2>&1 $stdout |\n" if $fVerbose; + open(PIPE, "$psql $quiet $echo -f $quoted_file_path $opts 2>&1 $stdout |") or die "exec '$psql': $OS_ERROR"; while () { @@ -115,7 +122,7 @@ sub InstallExtension chomp($sharedir); - RunSQLScript($db, "$sharedir/contrib/$ext", "Installing $ext extension ...", ''); + RunSQLScript($db, "$sharedir/contrib/$ext", "Installing $ext extension ..."); } sub CreateReplicationFunction @@ -265,8 +272,8 @@ sub CreateRelations push @opts, '--database', $databaseName; system($^X, "$FindBin::Bin/MBImport.pl", @opts, @$import); die "\nFailed to import dataset.\n" if ($CHILD_ERROR >> 8); - } else { - RunSQLScript($DB, 'InsertDefaultRows.sql', 'Adding default rows ...'); + } elsif ($initial_sql) { + RunSQLScript($DB, $initial_sql); } RunSQLScript($DB, 'CreatePrimaryKeys.sql', 'Creating primary keys ...'); @@ -473,6 +480,8 @@ sub Usage --install-extension Install a postgres extension module --extension-schema Which schema to install the extension module into. + --initial-sql SQL file to initialize the database with. + (default: admin/sql/InsertDefaultRows.sql) After the import option, you may specify one or more MusicBrainz data dump files for importing into the database. Once this script runs to completion @@ -506,6 +515,7 @@ sub Usage 'extension-schema=s' => \$fExtensionSchema, 'tmp-dir=s' => \$tmp_dir, 'reptype=s' => \$REPTYPE, + 'initial-sql=s' => \$initial_sql, ) or exit 2; my $DB = Databases->get($databaseName); diff --git a/admin/MBImport.pl b/admin/MBImport.pl index 8104204ba7e..ee796c8692e 100755 --- a/admin/MBImport.pl +++ b/admin/MBImport.pl @@ -10,6 +10,7 @@ use Getopt::Long; use DBDefs; use Sql; +use MusicBrainz::Script::Utils qw( find_mbdump_file ); use MusicBrainz::Server::Replication qw( :replication_type ); use MusicBrainz::Server::Constants qw( @FULL_TABLE_LIST ); @@ -328,7 +329,7 @@ sub empty sub ImportAllTables { for my $table (@$import_tables) { - my $file = (find_file($table))[0]; + my $file = find_mbdump_file($table, @ARGV); $file or print("No data file found for '$table', skipping\n"), next; $imported_tables{$table} = 1; @@ -365,27 +366,11 @@ sub ImportAllTables return 1; } -sub find_file -{ - my $table = shift; - my @r; - - for my $arg (@ARGV) - { - use File::Basename; - push(@r, $arg), next if -f $arg and basename($arg) eq $table; - push(@r, "$arg/$table"), next if -f "$arg/$table"; - push(@r, "$arg/mbdump/$table"), next if -f "$arg/mbdump/$table"; - } - - @r; -} - sub read_all_and_check { my $file = shift; - my @files = find_file($file); + my @files = find_mbdump_file($file, @ARGV); my %contents; my %uniq; diff --git a/admin/replication/ImportReplicationChanges b/admin/replication/ImportReplicationChanges index 6eb20718382..5db95318559 100755 --- a/admin/replication/ImportReplicationChanges +++ b/admin/replication/ImportReplicationChanges @@ -9,6 +9,7 @@ use FindBin; use lib "$FindBin::Bin/../../lib"; use Getopt::Long; +use MusicBrainz::Script::Utils qw( find_mbdump_file ); use MusicBrainz::Server::Context; use DBDefs; use Sql; @@ -194,7 +195,7 @@ sub ImportReplicationTables for my $table (('dbmirror_pending', 'dbmirror_pendingdata')) { - my $file = find_file($table); + my $file = find_mbdump_file($table, @ARGV); $file or print("No data file found for '$table', skipping\n"), die; if (not empty($table)) @@ -217,7 +218,7 @@ sub ImportReplicationTables sub ImportDBMirror2ReplicationTables { for my $table (qw( pending_data pending_keys pending_ts )) { - my $file = find_file($table); + my $file = find_mbdump_file($table, @ARGV); if (!$file) { # See NOTE-NOPK-1 in LoadReplicationChanges. @@ -236,21 +237,6 @@ sub ImportDBMirror2ReplicationTables { return 1; } -sub find_file -{ - my $table = shift; - - for my $arg (@ARGV) - { - use File::Basename; - return $arg if -f $arg and basename($arg) eq $table; - return "$arg/$table" if -f "$arg/$table"; - return "$arg/mbdump/$table" if -f "$arg/mbdump/$table"; - } - - undef; -} - { my @tmpdirs; diff --git a/docker/musicbrainz-tests/artwork-indexer.service b/docker/musicbrainz-tests/artwork-indexer.service index f53152843e5..1b3edf1ed60 100755 --- a/docker/musicbrainz-tests/artwork-indexer.service +++ b/docker/musicbrainz-tests/artwork-indexer.service @@ -2,4 +2,4 @@ cd /home/musicbrainz/artwork-indexer -exec sudo -E -H -u musicbrainz bash -c '. venv/bin/activate; exec python3 indexer.py --max-wait=1' +exec sudo -E -H -u musicbrainz bash -c '. venv/bin/activate; exec python3 indexer.py --config=config.selenium.ini --max-wait=1' diff --git a/docker/musicbrainz-tests/run_circleci_tests.sh b/docker/musicbrainz-tests/run_circleci_tests.sh index 84f4ce0b758..cd9e2dfe046 100755 --- a/docker/musicbrainz-tests/run_circleci_tests.sh +++ b/docker/musicbrainz-tests/run_circleci_tests.sh @@ -20,7 +20,7 @@ sudo -E -H -u musicbrainz cp docker/musicbrainz-tests/DBDefs.pm lib/ sv_start_if_down postgresql redis -sudo -E -H -u musicbrainz carton exec -- ./script/create_test_db.sh +REPLICATION_TYPE=1 sudo -E -H -u musicbrainz carton exec -- ./script/create_test_db.sh sudo -E -H -u musicbrainz make -C po test_source all_quiet deploy diff --git a/docker/musicbrainz-tests/run_selenium_tests.sh b/docker/musicbrainz-tests/run_selenium_tests.sh index 4f90038258d..48654e648f3 100755 --- a/docker/musicbrainz-tests/run_selenium_tests.sh +++ b/docker/musicbrainz-tests/run_selenium_tests.sh @@ -39,17 +39,11 @@ rabbitmqctl add_user sir sir rabbitmqctl add_vhost /sir-test rabbitmqctl set_permissions -p /sir-test sir '.*' '.*' '.*' -# Install the sir triggers into musicbrainz_selenium. +# Generate the sir extensions and triggers, which is required before +# invoking create_selenium_db.sh. export SIR_DIR=/home/musicbrainz/sir cd "$SIR_DIR" sudo -E -H -u musicbrainz sh -c '. venv/bin/activate; python -m sir amqp_setup; python -m sir extension; python -m sir triggers --broker-id=1' -psql -U postgres -f sql/CreateExtension.sql musicbrainz_selenium -psql -U musicbrainz -f sql/CreateFunctions.sql musicbrainz_selenium -psql -U musicbrainz -f sql/CreateTriggers.sql musicbrainz_selenium - -# Install the artwork_indexer schema into musicbrainz_selenium. -cd /home/musicbrainz/artwork-indexer -sudo -E -H -u musicbrainz sh -c '. venv/bin/activate; python indexer.py --setup-schema' cd /home/musicbrainz/musicbrainz-server diff --git a/docker/templates/Dockerfile.tests.m4 b/docker/templates/Dockerfile.tests.m4 index 4dbc7ced5ef..24edfba8521 100644 --- a/docker/templates/Dockerfile.tests.m4 +++ b/docker/templates/Dockerfile.tests.m4 @@ -167,7 +167,7 @@ COPY --chown=postgres:postgres \ RUN sudo -E -H -u postgres touch \ $PGDATA/pg_ident.conf -COPY docker/musicbrainz-tests/artwork-indexer-config.ini artwork-indexer/config.ini +COPY docker/musicbrainz-tests/artwork-indexer-config.ini artwork-indexer/config.selenium.ini COPY docker/musicbrainz-tests/artwork-redirect-config.ini artwork-redirect/config.ini COPY docker/musicbrainz-tests/sir-config.ini sir/config.ini diff --git a/lib/MusicBrainz/Script/Utils.pm b/lib/MusicBrainz/Script/Utils.pm index 407e8306c23..7fa6fa59fb6 100644 --- a/lib/MusicBrainz/Script/Utils.pm +++ b/lib/MusicBrainz/Script/Utils.pm @@ -3,18 +3,64 @@ use strict; use warnings; use English; +use List::AllUtils qw( uniq ); use feature 'state'; use base 'Exporter'; our @EXPORT_OK = qw( + find_files + find_mbdump_file get_primary_keys get_foreign_keys log retry ); +=sub find_files + +Looks for files named C<$file> in C<@search_paths>. The given paths may +contain a direct reference to the file, or directories which will be checked +instead. + +Returns an array of found files (in the specified search order). + +=cut + +sub find_files { + my ($file, @search_paths) = @_; + + return uniq(grep { -f } map { + my $search_path = $_; + ( + ($search_path =~ m/\Q$file\E$/ ? $search_path : ()), + "$search_path/$file" + ) + } @search_paths); +} + +=sub find_mbdump_file + +Looks for an mbdump file named C<$table> in C<@search_paths>. The semantics +are the same as for C, except: + + 1. The file is additionally searched for under an 'mbdump' sub-directory in + each search path. + 2. The first matching file is returned in scalar context. + +=cut + +sub find_mbdump_file { + my ($table, @search_paths) = @_; + + my @result = find_files($table, map { + ($_, "$_/mbdump") + } @search_paths); + + return wantarray ? @result : $result[0]; +} + =sub get_foreign_keys Get a list of foreign key columns for (C<$schema>, C<$table>). diff --git a/script/create_selenium_db.sh b/script/create_selenium_db.sh new file mode 100755 index 00000000000..38fd9ad77b8 --- /dev/null +++ b/script/create_selenium_db.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash + +set -o errexit -o nounset -o pipefail + +MB_SERVER_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../" && pwd) +cd "$MB_SERVER_ROOT" + +./script/create_test_db.sh SELENIUM + +./admin/psql SELENIUM < ./t/sql/selenium.sql + +if [[ $# -gt 0 ]]; then + EXTRA_SQL="$1" + if [[ -f "$EXTRA_SQL" ]]; then + ./admin/psql SELENIUM < "$EXTRA_SQL" + fi +fi + +DROP_SQL=$(cat <<'SQL' +\set ON_ERROR_STOP 1 +DROP EXTENSION IF EXISTS amqp CASCADE; +DROP SCHEMA IF EXISTS artwork_indexer CASCADE; +SQL +) +echo "$DROP_SQL" | ./admin/psql --system SELENIUM + +SIR_DIR="${SIR_DIR:="$MB_SERVER_ROOT"/../sir}" + +if [ -d "$SIR_DIR" ]; then + ./admin/psql --system SELENIUM < "$SIR_DIR"/sql/CreateExtension.sql + ./admin/psql SELENIUM < "$SIR_DIR"/sql/CreateFunctions.sql + ./admin/psql SELENIUM < "$SIR_DIR"/sql/CreateTriggers.sql +fi + +ARTWORK_INDEXER_DIR="${ARTWORK_INDEXER_DIR:="$MB_SERVER_ROOT"/../artwork-indexer}" + +if [ -d "$ARTWORK_INDEXER_DIR" ]; then + pushd "$ARTWORK_INDEXER_DIR" + VENV_DIR="$([ -d .venv ] && echo .venv || echo venv)" + . "$VENV_DIR"/bin/activate + python indexer.py --config=config.selenium.ini --setup-schema + popd +fi diff --git a/script/create_test_db.sh b/script/create_test_db.sh index fa630bc9a75..0a3cac46e08 100755 --- a/script/create_test_db.sh +++ b/script/create_test_db.sh @@ -12,100 +12,30 @@ fi source ./admin/functions.sh if ! script/database_exists $DATABASE; then - ./admin/InitDb.pl --createdb --database $DATABASE --clean + ./admin/InitDb.pl --createdb --database $DATABASE --clean --initial-sql 't/sql/initial.sql' +else + echo `date` : Clearing old test database + OUTPUT=` + echo " + DROP SCHEMA IF EXISTS musicbrainz CASCADE; + DROP SCHEMA IF EXISTS statistics CASCADE; + DROP SCHEMA IF EXISTS cover_art_archive CASCADE; + DROP SCHEMA IF EXISTS documentation CASCADE; + DROP SCHEMA IF EXISTS event_art_archive CASCADE; + DROP SCHEMA IF EXISTS report CASCADE; + DROP SCHEMA IF EXISTS wikidocs CASCADE; + DROP SCHEMA IF EXISTS sitemaps CASCADE; + DROP SCHEMA IF EXISTS json_dump CASCADE; + DROP SCHEMA IF EXISTS dbmirror2 CASCADE;" | ./admin/psql $DATABASE 2>&1 + ` || ( echo "$OUTPUT" && exit 1 ) + ./admin/InitDb.pl --database $DATABASE --clean --initial-sql 't/sql/initial.sql' fi -echo `date` : Clearing old test database -OUTPUT=` -echo " - DROP SCHEMA IF EXISTS musicbrainz CASCADE; - DROP SCHEMA IF EXISTS statistics CASCADE; - DROP SCHEMA IF EXISTS cover_art_archive CASCADE; - DROP SCHEMA IF EXISTS documentation CASCADE; - DROP SCHEMA IF EXISTS event_art_archive CASCADE; - DROP SCHEMA IF EXISTS wikidocs CASCADE; - DROP SCHEMA IF EXISTS sitemaps CASCADE; - DROP SCHEMA IF EXISTS json_dump CASCADE; - DROP SCHEMA IF EXISTS dbmirror2 CASCADE; - - CREATE SCHEMA musicbrainz; - CREATE SCHEMA statistics; - CREATE SCHEMA cover_art_archive; - CREATE SCHEMA documentation; - CREATE SCHEMA event_art_archive; - CREATE SCHEMA wikidocs; - CREATE SCHEMA sitemaps; - CREATE SCHEMA json_dump; - CREATE SCHEMA dbmirror2;" | ./admin/psql $DATABASE 2>&1 -` || ( echo "$OUTPUT" && exit 1 ) - -echo `date` : Creating MusicBrainz Schema -OUTPUT=`./admin/psql --system $DATABASE <./admin/sql/Extensions.sql 2>&1` || ( echo "$OUTPUT" && exit 1 ) -OUTPUT=`./admin/psql --system $DATABASE <./admin/sql/CreateSearchConfiguration.sql 2>&1` || ( echo "$OUTPUT" && exit 1 ) -OUTPUT=`./admin/psql $DATABASE <./admin/sql/CreateCollations.sql 2>&1` || ( echo "$OUTPUT" && exit 1 ) -OUTPUT=`./admin/psql $DATABASE <./admin/sql/CreateTypes.sql 2>&1` || ( echo "$OUTPUT" && exit 1 ) -OUTPUT=`./admin/psql $DATABASE <./admin/sql/CreateTables.sql 2>&1` || ( echo "$OUTPUT" && exit 1 ) -OUTPUT=`./admin/psql $DATABASE <./admin/sql/CreateViews.sql 2>&1` || ( echo "$OUTPUT" && exit 1 ) -OUTPUT=`./admin/psql $DATABASE <./admin/sql/CreateFunctions.sql 2>&1` || ( echo "$OUTPUT" && exit 1 ) -OUTPUT=`./admin/psql $DATABASE <./admin/sql/CreateConstraints.sql 2>&1` || ( echo "$OUTPUT" && exit 1 ) -OUTPUT=`./admin/psql $DATABASE <./admin/sql/CreatePrimaryKeys.sql 2>&1` || ( echo "$OUTPUT" && exit 1 ) -OUTPUT=`./admin/psql $DATABASE <./admin/sql/CreateFKConstraints.sql 2>&1` || ( echo "$OUTPUT" && exit 1 ) -OUTPUT=`./admin/psql $DATABASE <./admin/sql/CreateTriggers.sql 2>&1` || ( echo "$OUTPUT" && exit 1 ) -OUTPUT=`./admin/psql $DATABASE <./admin/sql/CreateIndexes.sql 2>&1` || ( echo "$OUTPUT" && exit 1 ) -OUTPUT=`./admin/psql $DATABASE <./admin/sql/CreateSearchIndexes.sql 2>&1` || ( echo "$OUTPUT" && exit 1 ) - -echo `date` : Creating Statistics Schema -OUTPUT=`./admin/psql $DATABASE <./admin/sql/statistics/CreateTables.sql 2>&1` || ( echo "$OUTPUT" && exit 1 ) -OUTPUT=`./admin/psql $DATABASE <./admin/sql/statistics/CreatePrimaryKeys.sql 2>&1` || ( echo "$OUTPUT" && exit 1 ) -OUTPUT=`./admin/psql $DATABASE <./admin/sql/statistics/CreateIndexes.sql 2>&1` || ( echo "$OUTPUT" && exit 1 ) - -echo `date` : Creating Cover Art Archive Schema -OUTPUT=`./admin/psql $DATABASE <./admin/sql/caa/CreateTables.sql 2>&1` || ( echo "$OUTPUT" && exit 1 ) -OUTPUT=`./admin/psql $DATABASE <./admin/sql/caa/CreateViews.sql 2>&1` || ( echo "$OUTPUT" && exit 1 ) -OUTPUT=`./admin/psql $DATABASE <./admin/sql/caa/CreateFunctions.sql 2>&1` || ( echo "$OUTPUT" && exit 1 ) -OUTPUT=`./admin/psql $DATABASE <./admin/sql/caa/CreatePrimaryKeys.sql 2>&1` || ( echo "$OUTPUT" && exit 1 ) -OUTPUT=`./admin/psql $DATABASE <./admin/sql/caa/CreateFKConstraints.sql 2>&1` || ( echo "$OUTPUT" && exit 1 ) -OUTPUT=`./admin/psql $DATABASE <./admin/sql/caa/CreateTriggers.sql 2>&1` || ( echo "$OUTPUT" && exit 1 ) -OUTPUT=`./admin/psql $DATABASE <./admin/sql/caa/CreateIndexes.sql 2>&1` || ( echo "$OUTPUT" && exit 1 ) - -echo `date` : Creating Wikidocs Schema -OUTPUT=`./admin/psql $DATABASE <./admin/sql/wikidocs/CreateTables.sql 2>&1` || ( echo "$OUTPUT" && exit 1 ) -OUTPUT=`./admin/psql $DATABASE <./admin/sql/wikidocs/CreatePrimaryKeys.sql 2>&1` || ( echo "$OUTPUT" && exit 1 ) - -echo `date` : Creating documentation Schema -OUTPUT=`./admin/psql $DATABASE <./admin/sql/documentation/CreateTables.sql 2>&1` || ( echo "$OUTPUT" && exit 1 ) -OUTPUT=`./admin/psql $DATABASE <./admin/sql/documentation/CreatePrimaryKeys.sql 2>&1` || ( echo "$OUTPUT" && exit 1 ) -OUTPUT=`./admin/psql $DATABASE <./admin/sql/documentation/CreateFKConstraints.sql 2>&1` || ( echo "$OUTPUT" && exit 1 ) - -echo `date` : Creating Event Art Archive Schema -OUTPUT=`./admin/psql $DATABASE <./admin/sql/eaa/CreateTables.sql 2>&1` || ( echo "$OUTPUT" && exit 1 ) -OUTPUT=`./admin/psql $DATABASE <./admin/sql/eaa/CreateViews.sql 2>&1` || ( echo "$OUTPUT" && exit 1 ) -OUTPUT=`./admin/psql $DATABASE <./admin/sql/eaa/CreateFunctions.sql 2>&1` || ( echo "$OUTPUT" && exit 1 ) -OUTPUT=`./admin/psql $DATABASE <./admin/sql/eaa/CreatePrimaryKeys.sql 2>&1` || ( echo "$OUTPUT" && exit 1 ) -OUTPUT=`./admin/psql $DATABASE <./admin/sql/eaa/CreateFKConstraints.sql 2>&1` || ( echo "$OUTPUT" && exit 1 ) -OUTPUT=`./admin/psql $DATABASE <./admin/sql/eaa/CreateTriggers.sql 2>&1` || ( echo "$OUTPUT" && exit 1 ) -OUTPUT=`./admin/psql $DATABASE <./admin/sql/eaa/CreateIndexes.sql 2>&1` || ( echo "$OUTPUT" && exit 1 ) - -echo `date` : Creating sitemaps Schema -OUTPUT=`./admin/psql $DATABASE <./admin/sql/sitemaps/CreateTables.sql 2>&1` || ( echo "$OUTPUT" && exit 1 ) -OUTPUT=`./admin/psql $DATABASE <./admin/sql/sitemaps/CreateFKConstraints.sql 2>&1` || ( echo "$OUTPUT" && exit 1 ) -OUTPUT=`./admin/psql $DATABASE <./admin/sql/sitemaps/CreateIndexes.sql 2>&1` || ( echo "$OUTPUT" && exit 1 ) - -echo `date` : Creating json_dump Schema -OUTPUT=`./admin/psql $DATABASE <./admin/sql/json_dump/CreateTables.sql 2>&1` || ( echo "$OUTPUT" && exit 1 ) -OUTPUT=`./admin/psql $DATABASE <./admin/sql/json_dump/CreatePrimaryKeys.sql 2>&1` || ( echo "$OUTPUT" && exit 1 ) -OUTPUT=`./admin/psql $DATABASE <./admin/sql/json_dump/CreateIndexes.sql 2>&1` || ( echo "$OUTPUT" && exit 1 ) - -echo `date` : Creating replication setup -OUTPUT=`./admin/psql $DATABASE <./admin/sql/ReplicationSetup.sql 2>&1` || ( echo "$OUTPUT" && exit 1 ) -OUTPUT=`./admin/psql $DATABASE <./admin/sql/dbmirror2/ReplicationSetup.sql 2>&1` || ( echo "$OUTPUT" && exit 1 ) - echo `date` : Set up pgtap extension -OUTPUT=`echo "CREATE EXTENSION pgtap WITH SCHEMA public;" | ./admin/psql --system $DATABASE 2>&1` || ( echo "$OUTPUT" && exit 1 ) - -echo `date` : Inserting initial data -OUTPUT=`./admin/psql $DATABASE < ./t/sql/initial.sql 2>&1` || ( echo "$OUTPUT" && exit 1 ) -OUTPUT=`./admin/psql $DATABASE <./admin/sql/SetSequences.sql 2>&1` || ( echo "$OUTPUT" && exit 1 ) +OUTPUT=`echo '\set ON_ERROR_STOP 1' $'\n' \ + 'CREATE EXTENSION IF NOT EXISTS pgtap WITH SCHEMA public;' | \ + ./admin/psql --system $DATABASE 2>&1` || \ + ( echo "$OUTPUT" && exit 1 ) echo `date` : Complete with no errors diff --git a/script/reset_selenium_env.sh b/script/reset_selenium_env.sh index 7d649b20bf3..efa91d9727a 100755 --- a/script/reset_selenium_env.sh +++ b/script/reset_selenium_env.sh @@ -11,6 +11,7 @@ SIR_REINDEX_LOG_FILE="$MBS_ROOT"/t/selenium/.sir-reindex.log terminate_pg_backends() { echo `date` : Terminating all PG backends local CANCEL_QUERY=$(cat <<'SQL' +\set ON_ERROR_STOP 1 SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE usename = 'musicbrainz' @@ -22,7 +23,6 @@ SQL if [[ $SIR_DIR ]]; then # Stop sir to avoid deadlocks below. - # TRUNCATE requires ACCESS EXCLUSIVE locks on each table. if [[ -f $SIR_PID_FILE ]]; then SIR_PID="$(cat "$SIR_PID_FILE")" fi @@ -35,11 +35,6 @@ if [[ $SIR_DIR ]]; then terminate_pg_backends - # Temporarily drop sir triggers to avoid queueing thousands of - # pending changes to the type tables via t/sql/initial.sql. - echo `date` : Dropping sir triggers - OUTPUT=`./admin/psql SELENIUM <"$SIR_DIR"/sql/DropTriggers.sql 2>&1` || ( echo "$OUTPUT" && exit 1 ) - echo `date` : Purging sir queues OUTPUT=`./script/purge_sir_queues.sh /sir-test 2>&1` || ( echo "$OUTPUT" && exit 1 ) @@ -49,28 +44,10 @@ else terminate_pg_backends fi -echo `date` : Truncating all tables -OUTPUT=`./admin/psql SELENIUM <./admin/sql/TruncateTables.sql 2>&1` || ( echo "$OUTPUT" && exit 1 ) -OUTPUT=`./admin/psql SELENIUM <./admin/sql/caa/TruncateTables.sql 2>&1` || ( echo "$OUTPUT" && exit 1 ) -OUTPUT=`./admin/psql SELENIUM <./admin/sql/eaa/TruncateTables.sql 2>&1` || ( echo "$OUTPUT" && exit 1 ) - -echo `date` : Inserting initial test data -OUTPUT=`./admin/psql SELENIUM < ./t/sql/initial.sql 2>&1` || ( echo "$OUTPUT" && exit 1 ) - -echo `date` : Setting sequences -OUTPUT=`./admin/psql SELENIUM <./admin/sql/SetSequences.sql 2>&1` || ( echo "$OUTPUT" && exit 1 ) - -echo `date` : Inserting Selenium test data -OUTPUT=`./admin/psql SELENIUM < ./t/sql/selenium.sql 2>&1` || ( echo "$OUTPUT" && exit 1 ) - -if [[ -f $EXTRA_SQL ]]; then - OUTPUT=`./admin/psql SELENIUM < "$EXTRA_SQL" 2>&1` || ( echo "$OUTPUT" && exit 1 ) -fi +echo `date` : Creating Selenium test database +OUTPUT=`./script/create_selenium_db.sh "$EXTRA_SQL" 2>&1` || ( echo "$OUTPUT" && exit 1 ) if [[ $SIR_DIR ]]; then - echo `date` : Creating sir triggers - OUTPUT=`./admin/psql SELENIUM <"$SIR_DIR"/sql/CreateTriggers.sql 2>&1` || ( echo "$OUTPUT" && exit 1 ) - pushd "$SIR_DIR" . venv/bin/activate diff --git a/script/setup_development_db.sh b/script/setup_development_db.sh deleted file mode 100755 index 74aefc249ca..00000000000 --- a/script/setup_development_db.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash - -set -o errexit -cd `dirname $0`/.. - -if [ -z "$1" ]; then - echo "==========================================================================" - echo " " - echo " WARNING: running this script will overwrite the production database." - echo " If you are sure you want to do this, run this:" - echo " " - echo " script/setup_development_db.sh --destroy-all-the-things" - echo " " - echo "==========================================================================" -elif [ "$1" = "--destroy-all-the-things" ]; then - - script/create_test_db.sh READWRITE - ( printf "BEGIN;\n"; cat ./t/sql/webservice.sql; printf "\nCOMMIT;\n"; ) | ./admin/psql READWRITE - ./admin/psql READWRITE < ./t/sql/editor.sql - ./admin/psql READWRITE < ./admin/sql/SetSequences.sql - -else - echo Unrecognized option "$1" -fi diff --git a/t/lib/t/MusicBrainz/Script/Utils.pm b/t/lib/t/MusicBrainz/Script/Utils.pm new file mode 100644 index 00000000000..a2197da3d13 --- /dev/null +++ b/t/lib/t/MusicBrainz/Script/Utils.pm @@ -0,0 +1,87 @@ +package t::MusicBrainz::Script::Utils; + +use strict; +use warnings; + +use File::Spec; +use File::Temp qw( tempdir ); +use Test::More; +use Test::Routine; + +use MusicBrainz::Script::Utils qw( find_files find_mbdump_file ); + +test 'find_files works as expected' => sub { + my $test = shift; + + my $tmp_dir = tempdir('find_files-XXXXXXXX', DIR => '/tmp', CLEANUP => 1); + my $sql_dir = File::Spec->catdir($tmp_dir, 'sql'); + my $caa_dir = File::Spec->catdir($sql_dir, 'caa'); + + system('mkdir', $sql_dir, $caa_dir); + + my $create_tables_sql_path = + File::Spec->catfile($caa_dir, 'CreateTables.sql'); + + system('touch', $create_tables_sql_path); + + my @result; + + @result = find_files( + 'caa/CreateTables.sql', + "$caa_dir/CreateTables.sql", + ); + is(scalar @result, 1, 'one file is found'); + is($result[0], $create_tables_sql_path, + 'file with path prefix is found by direct reference'); +}; + +test 'find_mbdump_file works as expected' => sub { + my $test = shift; + + my $dir1 = tempdir('find_mbdump_file-XXXXXXXX', DIR => '/tmp', CLEANUP => 1); + my $dir2 = File::Spec->catdir($dir1, 'mbdump'); + my $dir3 = File::Spec->catdir($dir1, 'recording'); + + system('mkdir', $dir2, $dir3); + + my $f1_path = File::Spec->catfile($dir1, 'artist'); + my $f2_path = File::Spec->catfile($dir2, 'recording'); + my $f3_path = File::Spec->catfile($dir2, 'artist'); + + system('touch', $f1_path, $f2_path, $f3_path); + + my ($result, @result); + + $result = find_mbdump_file('dne', $f1_path); + is($result, undef, + 'undef is returned for non-existent file in scalar context'); + + @result = find_mbdump_file('dne', $f1_path); + is(scalar @result, 0, + 'empty list is returned for non-existent file in list context'); + + $result = find_mbdump_file('recording', $dir3); + is($result, undef, + 'undef is returned for file matching directory name ' . + 'in scalar context'); + + $result = find_mbdump_file('artist', $f1_path); + is($result, $f1_path, 'scalar file is found by direct path'); + + $result = find_mbdump_file('artist', $dir1); + is($result, $f1_path, 'scalar file is found in directory'); + + $result = find_mbdump_file('recording', $dir1); + is($result, $f2_path, 'scalar file is found in mbdump sub-directory'); + + @result = find_mbdump_file('artist', $f1_path, $dir1, $dir2); + is(scalar @result, 2, 'two files are found (no duplicates)'); + is($result[0], $f1_path, 'first file is correctly found'); + is($result[1], $f3_path, 'second file is correctly found'); + + unlink $f1_path, $f2_path; + rmdir($dir1); + rmdir($dir2); +}; + +1; diff --git a/t/script/CheckSchemaMigration.sh b/t/script/CheckSchemaMigration.sh index 897bbc3d2d4..aa0d549a9cc 100755 --- a/t/script/CheckSchemaMigration.sh +++ b/t/script/CheckSchemaMigration.sh @@ -62,9 +62,13 @@ fi git restore --source=production -- admin/sql git restore --source=production -- admin/InitDb.pl git restore --source=production -- t/sql/initial.sql -./admin/InitDb.pl --database $DB2 --createdb --clean --reptype $REPLICATION_TYPE "${WITH_PENDING_ARGS[@]}" -./admin/psql $DB2 < t/sql/initial.sql -./admin/psql $DB2 < admin/sql/SetSequences.sql +./admin/InitDb.pl \ + --database $DB2 \ + --createdb \ + --clean \ + --reptype $REPLICATION_TYPE \ + --initial-sql t/sql/initial.sql \ + "${WITH_PENDING_ARGS[@]}" git restore admin/sql git restore t/sql git clean --force -- admin/sql diff --git a/t/script/ExportAllTables.t b/t/script/ExportAllTables.t index 791b76fa372..939f5494e90 100755 --- a/t/script/ExportAllTables.t +++ b/t/script/ExportAllTables.t @@ -117,6 +117,7 @@ test all => sub { '-U', $system_db->username, $test_db->database; + $ENV{REPLICATION_TYPE} = 1; # master system( File::Spec->catfile($root, 'admin/InitDb.pl'), '--database', 'TEST_FULL_EXPORT', @@ -129,12 +130,6 @@ test all => sub { File::Spec->catfile($output_dir, 'mbdump-event-art-archive.tar.bz2'), ); - my $replication_setup = File::Spec->catfile($root, 'admin/sql/ReplicationSetup.sql'); - system 'sh', '-c' => "$psql TEST_FULL_EXPORT < $replication_setup"; - - $replication_setup = File::Spec->catfile($root, 'admin/sql/dbmirror2/ReplicationSetup.sql'); - system 'sh', '-c' => "$psql TEST_FULL_EXPORT < $replication_setup"; - $exec_sql->(<<~"SQL"); SET client_min_messages TO WARNING; TRUNCATE replication_control CASCADE; diff --git a/t/sql/initial.sql b/t/sql/initial.sql index 4c56c14c34d..e0a5e149867 100644 --- a/t/sql/initial.sql +++ b/t/sql/initial.sql @@ -89,9 +89,7 @@ INSERT INTO instrument_type VALUES (3, 'Percussion instrument', NULL, 3, NULL, ' INSERT INTO instrument_type VALUES (4, 'Electronic instrument', NULL, 4, NULL, '98df6cec-95e5-3cbc-9a2d-7ea6c8be6f3c'); INSERT INTO instrument_type VALUES (5, 'Other instrument', NULL, 5, NULL, '01ba5777-02dd-347d-94a1-73e3db00215d'); -ALTER TABLE instrument DISABLE TRIGGER USER; INSERT INTO instrument VALUES (137, 'b3eac5f9-7859-4416-ac39-7154e2e8d348', 'piano', 2, 0, '2014-08-22 17:04:47.913121+00', '', ''); -ALTER TABLE instrument ENABLE TRIGGER USER; INSERT INTO instrument_alias VALUES (996, 137, 'Klavier', 'de', 0, '2014-08-20 17:28:48.384047+00', 1, 'Klavier', NULL, NULL, NULL, NULL, NULL, NULL, 't', 'f'); @@ -918,8 +916,8 @@ INSERT INTO release_alias_type VALUES (2, 'Search hint', NULL, 0, NULL, '02939c8 INSERT INTO release_group_alias_type VALUES (1, 'Release group name', NULL, 0, NULL, '156e24ca-8746-3cfc-99ae-0a867c765c67'); INSERT INTO release_group_alias_type VALUES (2, 'Search hint', NULL, 0, NULL, 'abc2db8a-7386-354d-82f4-252c0213cbe4'); -INSERT INTO release_group_primary_type VALUES (1, 'Album', NULL, 1, NULL, 'f529b476-6e62-324f-b0aa-1f3e33d313fc') ON CONFLICT (id) DO NOTHING; -INSERT INTO release_group_primary_type VALUES (2, 'Single', NULL, 2, NULL, 'd6038452-8ee0-3f68-affc-2de9a1ede0b9') ON CONFLICT (id) DO NOTHING; +INSERT INTO release_group_primary_type VALUES (1, 'Album', NULL, 1, NULL, 'f529b476-6e62-324f-b0aa-1f3e33d313fc'); +INSERT INTO release_group_primary_type VALUES (2, 'Single', NULL, 2, NULL, 'd6038452-8ee0-3f68-affc-2de9a1ede0b9'); INSERT INTO release_group_primary_type VALUES (3, 'EP', NULL, 3, NULL, '6d0c5bf6-7a33-3420-a519-44fc63eedebf'); INSERT INTO release_group_primary_type VALUES (11, 'Other', NULL, 99, NULL, '4fc3be2b-de1e-396b-a933-beb8f1607a22'); INSERT INTO release_group_primary_type VALUES (12, 'Broadcast', NULL, 4, NULL, '3b2e49e1-2875-37b8-9fa9-1f7cf3f49900'); @@ -942,7 +940,7 @@ INSERT INTO release_packaging VALUES (4, 'Cardboard/Paper Sleeve', NULL, 0, NULL INSERT INTO release_packaging VALUES (5, 'Other', NULL, 1, NULL, '815b7785-8284-3926-8f04-e48bc6c4d102'); INSERT INTO release_packaging VALUES (7, 'None', NULL, 2, NULL, '119eba76-b343-3e02-a292-f0f00644bb9b'); -INSERT INTO release_status VALUES (1, 'Official', NULL, 1, 'Any release officially sanctioned by the artist and/or their record company. Most releases will fit into this category.', '4e304316-386d-3409-af2e-78857eec5cfe') ON CONFLICT (id) DO NOTHING; +INSERT INTO release_status VALUES (1, 'Official', NULL, 1, 'Any release officially sanctioned by the artist and/or their record company. Most releases will fit into this category.', '4e304316-386d-3409-af2e-78857eec5cfe'); INSERT INTO release_status VALUES (2, 'Promotion', NULL, 2, 'A give-away release or a release intended to promote an upcoming official release (e.g. pre-release versions, releases included with a magazine, versions supplied to radio DJs for air-play).', '518ffc83-5cde-34df-8627-81bff5093d92'); INSERT INTO release_status VALUES (3, 'Bootleg', NULL, 3, 'An unofficial/underground release that was not sanctioned by the artist and/or the record company. This includes unofficial live recordings and pirated releases.', '1156806e-d06a-38bd-83f0-cf2284a808b9'); INSERT INTO release_status VALUES (4, 'Pseudo-Release', NULL, 4, 'An alternate version of a release where the titles have been changed. These don''t correspond to any real release and should be linked to the original release using the transl(iter)ation relationship.', '41121bb9-3413-3818-8a9a-9742318349aa'); @@ -1021,6 +1019,6 @@ INSERT INTO work_type VALUES (28, 'Play', NULL, 2, 'A play is a form of literatu INSERT INTO work_type VALUES (29, 'Musical', NULL, 2, 'Musical theatre is a form of theatrical performance that combines songs, spoken dialogue, acting, and dance.', '9ca5e067-acf7-3cd6-baa4-92bf1975bf24'); INSERT INTO work_type VALUES (30, 'Incidental music', NULL, 2, 'Incidental music is music written as background for (usually) a theatre play.', '3cd7c402-444a-3d04-a154-4fa7d13e4ec6'); -INSERT INTO editor (id, name, privs, email, bio, email_confirm_date, password, ha1) VALUES (4, 'ModBot', 2, 'support@musicbrainz.org', 'ModBot is a bot used by the MusicBrainz Server to perform a variety of automated functions. \r+', '2013-07-26 11:48:31.088042+00', '{CLEARTEXT}mb', '03503a81a03bdbb6055f4a6c8b86b5b8') ON CONFLICT (id) DO NOTHING; +INSERT INTO editor (id, name, privs, email, bio, email_confirm_date, password, ha1) VALUES (4, 'ModBot', 2, 'support@musicbrainz.org', 'ModBot is a bot used by the MusicBrainz Server to perform a variety of automated functions. \r+', '2013-07-26 11:48:31.088042+00', '{CLEARTEXT}mb', '03503a81a03bdbb6055f4a6c8b86b5b8'); COMMIT; diff --git a/t/tests.t b/t/tests.t index 0feba9721dd..ad7b1443bbc 100644 --- a/t/tests.t +++ b/t/tests.t @@ -14,6 +14,7 @@ my @classes = ( 't::MusicBrainz::DataStore::RedisMulti', 't::MusicBrainz::Script::RemoveEmptyURLs', 't::MusicBrainz::Script::RemoveUnreferencedRows', + 't::MusicBrainz::Script::Utils', map { Module::Pluggable::Object->new( search_path => $_ )->plugins } (