From 1bd5d7d1b86d3d6470552ba4979d850377977864 Mon Sep 17 00:00:00 2001 From: Dave Miller Date: Sun, 28 Apr 2024 00:25:47 -0400 Subject: [PATCH 1/3] Bug 1467006: Explicitly support MariaDB (#100) Also fixes the following: * Bug 868869: Require MariaDB 10.0.5 * Bug 868867: Require DBD::mysql 4.032 --- .github/workflows/ci.yml | 11 + Bugzilla/Constants.pm | 19 +- Bugzilla/DB.pm | 29 +- Bugzilla/DB/MariaDB.pm | 1093 +++++++++++++++++ Bugzilla/DB/Schema/MariaDB.pm | 463 +++++++ Bugzilla/DaemonControl.pm | 1 - Bugzilla/Install/Requirements.pm | 8 +- Dockerfile | 2 +- Makefile.PL | 9 +- cpanfile | 4 +- docker-compose.test-mariadb.yml | 68 + docker/gen-bugzilla-slim-mariadb106.sh | 113 ++ docker/images/Dockerfile.bugzilla-mariadb106 | 57 + docker/run-tests-in-docker.sh | 2 + extensions/AntiSpam/Extension.pm | 6 +- .../template/en/default/config.json.tmpl | 4 +- scripts/cpanfile_fixed_versions.pl | 4 +- t/004template.t | 4 +- template/en/default/config.js.tmpl | 4 +- template/en/default/setup/strings.txt.pl | 2 +- 20 files changed, 1878 insertions(+), 25 deletions(-) create mode 100644 Bugzilla/DB/MariaDB.pm create mode 100644 Bugzilla/DB/Schema/MariaDB.pm create mode 100644 docker-compose.test-mariadb.yml create mode 100644 docker/gen-bugzilla-slim-mariadb106.sh create mode 100644 docker/images/Dockerfile.bugzilla-mariadb106 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0b42b44a4a..ade4be0259 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,6 +37,17 @@ jobs: - name: Run bmo specific tests run: docker compose -f docker-compose.test.yml run -e CI=1 bugzilla6.test test_bmo -q -f t/bmo/*.t + test_bugzilla6_mariadb: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Install docker-compose + run: sudo apt update && sudo apt install -y docker-compose + - name: Build the Docker images + run: docker compose -f docker-compose.test-mariadb.yml build + - name: Run bmo specific tests + run: docker compose -f docker-compose.test-mariadb.yml run -e CI=1 bugzilla6.test test_bmo -q -f t/bmo/*.t + test_bugzilla6_pg: runs-on: ubuntu-latest steps: diff --git a/Bugzilla/Constants.pm b/Bugzilla/Constants.pm index 22b2c4d880..71e20bfee9 100644 --- a/Bugzilla/Constants.pm +++ b/Bugzilla/Constants.pm @@ -544,15 +544,30 @@ use constant DB_MODULE => { module => 'DBD::mysql', # Disallow development versions - blacklist => ['_'], + blocklist => ['_'], # For UTF-8 support. 4.001 makes sure that blobs aren't # marked as UTF-8. - version => '4.001', + version => '4.032', }, name => 'MySQL' }, + # MariaDB used to be a drop-in replacement for MySQL but now it + # isn't so we have a separate driver + 'mariadb' => { + db => 'Bugzilla::DB::MariaDB', + db_version => '10.0.5', + dbd => { + package => 'DBD-MariaDB', + module => 'DBD::MariaDB', + + # Disallow development versions + blocklist => ['_'], + }, + name => 'MariaDB' + }, + # Also see Bugzilla::DB::Pg::bz_check_server_version, which has special # code to require DBD::Pg 2.17.2 for PostgreSQL 9 and above. 'pg' => { diff --git a/Bugzilla/DB.pm b/Bugzilla/DB.pm index 3e9d4cc087..c71e5f10e0 100644 --- a/Bugzilla/DB.pm +++ b/Bugzilla/DB.pm @@ -276,22 +276,45 @@ sub bz_check_server_version { my ($self, $db, $output) = @_; my $sql_vers = $self->bz_server_version; - + if (((lc($db->{name}) eq 'mysql') || (lc($db->{name}) eq "mariadb")) + && ($sql_vers =~ s/^5\.5\.5-//)) { + # Version 5.5.5 of MySQL never existed. MariaDB >= 10 always puts '5.5.5-' + # at the front of its version string to get around a limitation in the + # replication protocol it shares with MySQL. So if the version starts with + # '5.5.5-' then we can assume this is MariaDB and the real version number + # will immediately follow that. + $db = DB_MODULE->{'mariadb'}; + } + my $sql_dontwant = exists $db->{db_blocklist} ? $db->{db_blocklist} : []; my $sql_want = $db->{db_version}; my $version_ok = vers_cmp($sql_vers, $sql_want) > -1 ? 1 : 0; - + my $blocklisted; + if ($version_ok) { + $blocklisted = grep($sql_vers =~ /$_/, @$sql_dontwant); + $version_ok = 0 if $blocklisted; + } my $sql_server = $db->{name}; if ($output) { Bugzilla::Install::Requirements::_checking_for({ package => $sql_server, wanted => $sql_want, found => $sql_vers, - ok => $version_ok + ok => $version_ok, + blocklisted => $blocklisted }); } # Check what version of the database server is installed and let # the user know if the version is too old to be used with Bugzilla. + if ($blocklisted) { + die < and L. + +=cut + +package Bugzilla::DB::MariaDB; + +use 5.10.1; +use Moo; + +extends qw(Bugzilla::DB); + +use Bugzilla::Constants; +use Bugzilla::Install::Util qw(install_string); +use Bugzilla::Util; +use Bugzilla::Error; +use Bugzilla::DB::Schema::MariaDB; + +use List::Util qw(max any all); +use Text::ParseWords; +use Carp; + +# This is how many comments of MAX_COMMENT_LENGTH we expect on a single bug. +# In reality, you could have a LOT more comments than this, because +# MAX_COMMENT_LENGTH is big. +use constant MAX_COMMENTS => 50; + +use constant FULLTEXT_OR => '|'; + +sub BUILDARGS { + my ($class, $params) = @_; + my ($user, $pass, $host, $dbname, $port, $sock) + = @$params{qw(db_user db_pass db_host db_name db_port db_sock)}; + + # construct the DSN from the parameters we got + my $dsn = "dbi:MariaDB:host=$host;database=$dbname"; + $dsn .= ";port=$port" if $port; + $dsn .= ";mariadb_socket=$sock" if $sock; + + my %attrs = (); + + # MariaDB SSL options + my ($ssl_ca_file, $ssl_ca_path, $ssl_cert, $ssl_key) = + @$params{qw(db_mysql_ssl_ca_file db_mysql_ssl_ca_path + db_mysql_ssl_client_cert db_mysql_ssl_client_key)}; + if ($ssl_ca_file || $ssl_ca_path || $ssl_cert || $ssl_key) { + $attrs{'mariadb_ssl'} = 1; + $attrs{'mariadb_ssl_ca_file'} = $ssl_ca_file if $ssl_ca_file; + $attrs{'mariadb_ssl_ca_path'} = $ssl_ca_path if $ssl_ca_path; + $attrs{'mariadb_ssl_client_cert'} = $ssl_cert if $ssl_cert; + $attrs{'mariadb_ssl_client_key'} = $ssl_key if $ssl_key; + } + + return {dsn => $dsn, user => $user, pass => $pass, attrs => \%attrs}; +} + +sub on_dbi_connected { + my ($class, $dbh) = @_; + + # This makes sure that if the tables are encoded as UTF-8, we + # return their data correctly. + my $charset = $class->utf8_charset; + my $collate = $class->utf8_collate; + $dbh->do("SET NAMES $charset COLLATE $collate"); + + # Bug 321645 - disable MySQL strict mode, if set + my ($var, $sql_mode) + = $dbh->selectrow_array("SHOW VARIABLES LIKE 'sql\\_mode'"); + + if ($sql_mode) { + + # STRICT_TRANS_TABLE or STRICT_ALL_TABLES enable MySQL strict mode, + # causing bug 321645. TRADITIONAL sets these modes (among others) as + # well, so it has to be stipped as well + my $new_sql_mode = join(",", + grep { $_ !~ /^STRICT_(?:TRANS|ALL)_TABLES|TRADITIONAL$/ } + split(/,/, $sql_mode)); + + if ($sql_mode ne $new_sql_mode) { + $dbh->do("SET SESSION sql_mode = ?", undef, $new_sql_mode); + } + } + + # Allow large GROUP_CONCATs (largely for inserting comments + # into bugs_fulltext). + $dbh->do('SET SESSION group_concat_max_len = 128000000'); +} + +# when last_insert_id() is supported on MySQL by lowest DBI/DBD version +# required by Bugzilla, this implementation can be removed. +sub bz_last_key { + my ($self) = @_; + + my ($last_insert_id) = $self->selectrow_array('SELECT LAST_INSERT_ID()'); + + return $last_insert_id; +} + +sub sql_group_concat { + my ($self, $column, $separator, $sort) = @_; + $separator = $self->quote(', ') if !defined $separator; + $sort = 1 if !defined $sort; + if ($sort) { + my $sort_order = $column; + $sort_order =~ s/^DISTINCT\s+//i; + $column = "$column ORDER BY $sort_order"; + } + return "GROUP_CONCAT($column SEPARATOR $separator)"; +} + +sub sql_regexp { + my ($self, $expr, $pattern, $nocheck, $real_pattern) = @_; + $real_pattern ||= $pattern; + + $self->bz_check_regexp($real_pattern) if !$nocheck; + + return "$expr REGEXP $pattern"; +} + +sub sql_not_regexp { + my ($self, $expr, $pattern, $nocheck, $real_pattern) = @_; + $real_pattern ||= $pattern; + + $self->bz_check_regexp($real_pattern) if !$nocheck; + + return "$expr NOT REGEXP $pattern"; +} + +sub sql_limit { + my ($self, $limit, $offset) = @_; + + if (defined($offset)) { + return "LIMIT $offset, $limit"; + } + else { + return "LIMIT $limit"; + } +} + +sub sql_string_concat { + my ($self, @params) = @_; + + return 'CONCAT(' . join(', ', @params) . ')'; +} + +sub sql_fulltext_search { + my ($self, $column, $text) = @_; + + # Add the boolean mode modifier if the search string contains + # boolean operators at the start or end of a word. + my $mode = ''; + if ($text =~ /(?:^|\W)[+\-<>~"()]/ || $text =~ /[()"*](?:$|\W)/) { + $mode = 'IN BOOLEAN MODE'; + + my @terms = split(quotemeta(FULLTEXT_OR), $text); + foreach my $term (@terms) { + + # quote un-quoted compound words + my @words = grep {defined} quotewords('[\s()]+', 'delimiters', $term); + foreach my $word (@words) { + + # match words that have non-word chars in the middle of them + if ($word =~ /\w\W+\w/ && $word !~ m/"/) { + $word = '"' . $word . '"'; + + # match words that contain only boolean operators + } + elsif ($word =~ /^[\+\-\<\>\~\*]+$/) { + $word = '"' . $word . '"'; + } + } + $term = join('', @words); + } + $text = join(FULLTEXT_OR, @terms); + } + + # quote the text for use in the MATCH AGAINST expression + $text = $self->quote($text); + + return "MATCH($column) AGAINST($text $mode)"; +} + +sub sql_istring { + my ($self, $string) = @_; + + return $string; +} + +sub sql_from_days { + my ($self, $days) = @_; + + return "FROM_DAYS($days)"; +} + +sub sql_to_days { + my ($self, $date) = @_; + + return "TO_DAYS($date)"; +} + +sub sql_date_format { + my ($self, $date, $format) = @_; + + $format = "%Y-%m-%d %H:%i:%s" if !$format; + + return "DATE_FORMAT($date, " . $self->quote($format) . ")"; +} + +sub sql_date_math { + my ($self, $date, $operator, $interval, $units) = @_; + + return "$date $operator INTERVAL $interval $units"; +} + +sub sql_iposition { + my ($self, $fragment, $text) = @_; + return "INSTR($text, $fragment)"; +} + +sub sql_position { + my ($self, $fragment, $text) = @_; + + return "INSTR(CAST($text AS BINARY), CAST($fragment AS BINARY))"; +} + +sub sql_group_by { + my ($self, $needed_columns, $optional_columns) = @_; + + # MySQL allows you to specify the minimal subset of columns to get + # a unique result. While it does allow specifying all columns as + # ANSI SQL requires, according to MySQL documentation, the fewer + # columns you specify, the faster the query runs. + return "GROUP BY $needed_columns"; +} + +sub sql_prefix_match_fulltext { + my ($self, $column, $prefix) = @_; + my @words = split(/\s+/, $prefix); + if (all {/^\w+$/} @words) { + $words[-1] .= '*'; + return sprintf('MATCH(%s) AGAINST (%s IN BOOLEAN MODE)', + $column, $self->quote(join(' ', map {"+$_"} @words))); + } + else { + return sprintf('MATCH(%s) AGAINST (%s)', $column, $self->quote($prefix)); + } +} + +sub bz_explain { + my ($self, $sql) = @_; + my $sth = $self->prepare("EXPLAIN $sql"); + $sth->execute(); + my $columns = $sth->{'NAME'}; + my $lengths = $sth->{'mysql_max_length'}; + my $format_string = '|'; + my $i = 0; + foreach my $column (@$columns) { + + # Sometimes the column name is longer than the contents. + my $length = max($lengths->[$i], length($column)); + $format_string .= ' %-' . $length . 's |'; + $i++; + } + + my $first_row = sprintf($format_string, @$columns); + my @explain_rows = ($first_row, '-' x length($first_row)); + while (my $row = $sth->fetchrow_arrayref) { + my @fixed = map { defined $_ ? $_ : 'NULL' } @$row; + push(@explain_rows, sprintf($format_string, @fixed)); + } + + return join("\n", @explain_rows); +} + +sub _bz_get_initial_schema { + my ($self) = @_; + return $self->_bz_build_schema_from_disk(); +} + +##################################################################### +# Database Setup +##################################################################### + +sub bz_check_server_version { + my $self = shift; + + my $lc = Bugzilla->localconfig; + if (lc(Bugzilla->localconfig->db_name) eq 'mysql') { + die "It is not safe to run Bugzilla inside a database named 'mysql'.\n" + . " Please pick a different value for \$db_name in localconfig.\n"; + } + + $self->SUPER::bz_check_server_version(@_); +} + +sub bz_setup_database { + my ($self) = @_; + + # The "comments" field of the bugs_fulltext table could easily exceed + # MySQL's default max_allowed_packet. Also, MySQL should never have + # a max_allowed_packet smaller than our max_attachment_size. So, we + # warn the user here if max_allowed_packet is too small. + my $min_max_allowed = MAX_COMMENTS * MAX_COMMENT_LENGTH; + my (undef, $current_max_allowed) + = $self->selectrow_array(q{SHOW VARIABLES LIKE 'max\_allowed\_packet'}); + + # This parameter is not yet defined when the DB is being built for + # the very first time. The code below still works properly, however, + # because the default maxattachmentsize is smaller than $min_max_allowed. + my $max_attachment = (Bugzilla->params->{'maxattachmentsize'} || 0) * 1024; + my $needed_max_allowed = max($min_max_allowed, $max_attachment); + if ($current_max_allowed < $needed_max_allowed) { + warn install_string('max_allowed_packet', + {current => $current_max_allowed, needed => $needed_max_allowed}) + . "\n"; + } + + # Make sure the installation has InnoDB turned on, or we're going to be + # doing silly things like making foreign keys on MyISAM tables, which is + # hard to fix later. We do this up here because none of the code below + # works if InnoDB is off. (Particularly if we've already converted the + # tables to InnoDB.) + my %engines = @{$self->selectcol_arrayref('SHOW ENGINES', {Columns => [1, 2]})}; + if (!$engines{InnoDB} || $engines{InnoDB} !~ /^(YES|DEFAULT)$/) { + die install_string('mysql_innodb_disabled'); + } + + if ($self->utf8_charset eq 'utf8mb4') { + my %global = map {@$_} + @{$self->selectall_arrayref(q(SHOW GLOBAL VARIABLES LIKE 'innodb_%'))}; + + # In versions of MySQL > 8, the default value for innodb_file_format is Barracuda + # and the setting was deprecated. Also innodb_file_per_table also now defaults + # to ON. innodb_large_prefix has also been removed in newer MySQL versions. + my $utf8mb4_supported + = (!exists $global{innodb_file_format} + || $global{innodb_file_format} eq 'Barracuda') + && (!exists $global{innodb_file_per_table} + || $global{innodb_file_per_table} eq 'ON') + && (!exists $global{innodb_large_prefix} + || $global{innodb_large_prefix} eq 'ON'); + + die install_string('mysql_innodb_settings') unless $utf8mb4_supported; + + my $tables = $self->selectall_arrayref('SHOW TABLE STATUS'); + foreach my $table (@$tables) { + my ($table, undef, undef, $row_format) = @$table; + my $new_row_format = $self->default_row_format($table); + next if $new_row_format =~ /compact/i; + if (lc($new_row_format) ne lc($row_format)) { + print install_string( + 'mysql_row_format_conversion', {table => $table, format => $new_row_format} + ), + "\n"; + $self->do( + sprintf 'ALTER TABLE %s ROW_FORMAT=%s', + $self->quote_identifier($table), + $new_row_format + ); + } + } + } + + my ($sd_index_deleted, $longdescs_index_deleted); + my @tables = $self->bz_table_list_real(); + + # We want to convert tables to InnoDB, but it's possible that they have + # fulltext indexes on them, and conversion will fail unless we remove + # the indexes. + if (grep($_ eq 'bugs', @tables) and !grep($_ eq 'bugs_fulltext', @tables)) { + if ($self->bz_index_info_real('bugs', 'short_desc')) { + $self->bz_drop_index_raw('bugs', 'short_desc'); + } + if ($self->bz_index_info_real('bugs', 'bugs_short_desc_idx')) { + $self->bz_drop_index_raw('bugs', 'bugs_short_desc_idx'); + $sd_index_deleted = 1; # Used for later schema cleanup. + } + } + if (grep($_ eq 'longdescs', @tables) and !grep($_ eq 'bugs_fulltext', @tables)) + { + if ($self->bz_index_info_real('longdescs', 'thetext')) { + $self->bz_drop_index_raw('longdescs', 'thetext'); + } + if ($self->bz_index_info_real('longdescs', 'longdescs_thetext_idx')) { + $self->bz_drop_index_raw('longdescs', 'longdescs_thetext_idx'); + $longdescs_index_deleted = 1; # For later schema cleanup. + } + } + + # Upgrade tables from MyISAM to InnoDB + my $db_name = Bugzilla->localconfig->db_name; + my $myisam_tables = $self->selectcol_arrayref( + 'SELECT TABLE_NAME FROM information_schema.TABLES + WHERE TABLE_SCHEMA = ? AND ENGINE = ?', undef, $db_name, 'MyISAM' + ); + + if (scalar @$myisam_tables) { + print "Bugzilla now uses the InnoDB storage engine in MariaDB for", + " most tables.\nConverting tables to InnoDB:\n"; + foreach my $table (@$myisam_tables) { + print "Converting table $table... "; + $self->do('ALTER TABLE ' . $self->quote_identifier($table) . ' ENGINE = InnoDB'); + print "done.\n"; + } + } + + # Versions of Bugzilla before the existence of Bugzilla::DB::Schema did + # not provide explicit names for the table indexes. This means + # that our upgrades will not be reliable, because we look for the name + # of the index, not what fields it is on, when doing upgrades. + # (using the name is much better for cross-database compatibility + # and general reliability). It's also very important that our + # Schema object be consistent with what is on the disk. + # + # While we're at it, we also fix some inconsistent index naming + # from the original checkin of Bugzilla::DB::Schema. + + # We check for the existence of a particular "short name" index that + # has existed at least since Bugzilla 2.8, and probably earlier. + # For fixing the inconsistent naming of Schema indexes, + # we also check for one of those inconsistently-named indexes. + if ( + grep($_ eq 'bugs', @tables) + && ( $self->bz_index_info_real('bugs', 'assigned_to') + || $self->bz_index_info_real('flags', 'flags_bidattid_idx')) + ) + { + + # This is a check unrelated to the indexes, to see if people are + # upgrading from 2.18 or below, but somehow have a bz_schema table + # already. This only happens if they have done a mysqldump into + # a database without doing a DROP DATABASE first. + # We just do the check here since this check is a reliable way + # of telling that we are upgrading from a version pre-2.20. + if (grep($_ eq 'bz_schema', $self->bz_table_list_real())) { + die install_string('bz_schema_exists_before_220'); + } + + my $bug_count = $self->selectrow_array("SELECT COUNT(*) FROM bugs"); + + # We estimate one minute for each 3000 bugs, plus 3 minutes just + # to handle basic MySQL stuff. + my $rename_time = int($bug_count / 3000) + 3; + + # And 45 minutes for every 15,000 attachments, per some experiments. + my ($attachment_count) + = $self->selectrow_array("SELECT COUNT(*) FROM attachments"); + $rename_time += int(($attachment_count * 45) / 15000); + + # If we're going to take longer than 5 minutes, we let the user know + # and allow them to abort. + if ($rename_time > 5) { + print "\n", install_string('mysql_index_renaming', {minutes => $rename_time}); + + # Wait 45 seconds for them to respond. + sleep(45) unless Bugzilla->installation_answers->{NO_PAUSE}; + } + print "Renaming indexes...\n"; + + # We can't be interrupted, because of how the "if" + # works above. + local $SIG{INT} = 'IGNORE'; + local $SIG{TERM} = 'IGNORE'; + local $SIG{PIPE} = 'IGNORE'; + + # Certain indexes had names in Schema that did not easily conform + # to a standard. We store those names here, so that they + # can be properly renamed. + # Also, sometimes an old mysqldump would incorrectly rename + # unique indexes to "PRIMARY", so we address that here, also. + my $bad_names = { + + # 'when' is a possible leftover from Bugzillas before 2.8 + bugs_activity => + ['when', 'bugs_activity_bugid_idx', 'bugs_activity_bugwhen_idx'], + cc => ['PRIMARY'], + longdescs => ['longdescs_bugid_idx', 'longdescs_bugwhen_idx'], + flags => ['flags_bidattid_idx'], + flaginclusions => ['flaginclusions_tpcid_idx'], + flagexclusions => ['flagexclusions_tpc_id_idx'], + keywords => ['PRIMARY'], + milestones => ['PRIMARY'], + profiles_activity => ['profiles_activity_when_idx'], + group_control_map => ['group_control_map_gid_idx', 'PRIMARY'], + user_group_map => ['PRIMARY'], + group_group_map => ['PRIMARY'], + email_setting => ['PRIMARY'], + bug_group_map => ['PRIMARY'], + category_group_map => ['PRIMARY'], + watch => ['PRIMARY'], + namedqueries => ['PRIMARY'], + series_data => ['PRIMARY'], + + # series_categories is dealt with below, not here. + }; + + # The series table is broken and needs to have one index + # dropped before we begin the renaming, because it had a + # useless index on it that would cause a naming conflict here. + if (grep($_ eq 'series', @tables)) { + my $dropname; + + # This is what the bad index was called before Schema. + if ($self->bz_index_info_real('series', 'creator_2')) { + $dropname = 'creator_2'; + } + + # This is what the bad index is called in Schema. + elsif ($self->bz_index_info_real('series', 'series_creator_idx')) { + $dropname = 'series_creator_idx'; + } + $self->bz_drop_index_raw('series', $dropname) if $dropname; + } + + # The email_setting table also had the same problem. + if (grep($_ eq 'email_setting', @tables) + && $self->bz_index_info_real('email_setting', 'email_settings_user_id_idx')) + { + $self->bz_drop_index_raw('email_setting', 'email_settings_user_id_idx'); + } + + # Go through all the tables. + foreach my $table (@tables) { + + # Will contain the names of old indexes as keys, and the + # definition of the new indexes as a value. The values + # include an extra hash key, NAME, with the new name of + # the index. + my %rename_indexes; + + # And go through all the columns on each table. + my @columns = $self->bz_table_columns_real($table); + + # We also want to fix the silly naming of unique indexes + # that happened when we first checked-in Bugzilla::DB::Schema. + if ($table eq 'series_categories') { + + # The series_categories index had a nonstandard name. + push(@columns, 'series_cats_unique_idx'); + } + elsif ($table eq 'email_setting') { + + # The email_setting table had a similar problem. + push(@columns, 'email_settings_unique_idx'); + } + else { + push(@columns, "${table}_unique_idx"); + } + + # And this is how we fix the other inconsistent Schema naming. + push(@columns, @{$bad_names->{$table}}) if (exists $bad_names->{$table}); + foreach my $column (@columns) { + + # If we have an index named after this column, it's an + # old-style-name index. + if (my $index = $self->bz_index_info_real($table, $column)) { + + # Fix the name to fit in with the new naming scheme. + $index->{NAME} = $table . "_" . $index->{FIELDS}->[0] . "_idx"; + print "Renaming index $column to " . $index->{NAME} . "...\n"; + $rename_indexes{$column} = $index; + } # if + } # foreach column + + my @rename_sql + = $self->_bz_schema->get_rename_indexes_ddl($table, %rename_indexes); + $self->do($_) foreach (@rename_sql); + + } # foreach table + } # if old-name indexes + + # If there are no tables, but the DB isn't UTF-8 and it should be, + # then we should alter the database to be UTF-8. We know it should be + # if the UTF-8 parameter is true or there are no params at all. + # This kind of situation happens when people create the database + # themselves, and if we don't do this they will get the big + # scary WARNING statement about conversion to UTF8. + unless ($self->bz_db_is_utf8) { + $self->_alter_db_charset_to_utf8(); + } + + # And now we create the tables and the Schema object. + $self->SUPER::bz_setup_database(); + + if ($sd_index_deleted) { + $self->_bz_real_schema->delete_index('bugs', 'bugs_short_desc_idx'); + $self->_bz_store_real_schema; + } + if ($longdescs_index_deleted) { + $self->_bz_real_schema->delete_index('longdescs', 'longdescs_thetext_idx'); + $self->_bz_store_real_schema; + } + + # 2005-09-24 - bugreport@peshkin.net, bug 307602 + # Make sure that default 4G table limit is overridden + my $attach_data_create = $self->selectrow_array( + 'SELECT CREATE_OPTIONS FROM information_schema.TABLES + WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ?', undef, $db_name, 'attach_data' + ); + if ($attach_data_create !~ /MAX_ROWS/i) { + print "Converting attach_data maximum size to 100G...\n"; + $self->do( + "ALTER TABLE attach_data + AVG_ROW_LENGTH=1000000, + MAX_ROWS=100000" + ); + } + + # Convert the database to UTF-8 if the UTF-8 parameter is on. + # We check if any table isn't UTF-8, because lots of crazy + # partial-conversion situations can happen, and this handles anything + # that could come up (including having the DB charset be UTF-8 but not + # the table charsets. + # + # TABLE_COLLATION IS NOT NULL prevents us from trying to convert views. + my $charset = $self->utf8_charset; + my $collate = $self->utf8_collate; + my $non_utf8_tables = $self->selectrow_array( + "SELECT 1 FROM information_schema.TABLES + WHERE TABLE_SCHEMA = ? AND TABLE_COLLATION IS NOT NULL + AND TABLE_COLLATION != ? + LIMIT 1", undef, $db_name, $collate + ); + + if (Bugzilla->params->{'utf8'} && $non_utf8_tables) { + print "\n", install_string('mysql_utf8_conversion'); + + if (!Bugzilla->installation_answers->{NO_PAUSE}) { + if (Bugzilla->installation_mode == INSTALLATION_MODE_NON_INTERACTIVE) { + die install_string('continue_without_answers'), "\n"; + } + else { + print "\n " . install_string('enter_or_ctrl_c'); + getc; + } + } + + print + "Converting table storage format to $charset (collate $collate). This may take a while.\n"; + foreach my $table ($self->bz_table_list_real) { + my $info_sth = $self->prepare("SHOW FULL COLUMNS FROM $table"); + $info_sth->execute(); + my (@binary_sql, @utf8_sql); + while (my $column = $info_sth->fetchrow_hashref) { + + # Our conversion code doesn't work on enum fields, but they + # all go away later in checksetup anyway. + next if $column->{Type} =~ /enum/i; + + # If this particular column isn't stored in UTF-8 + if ( $column->{Collation} + && $column->{Collation} ne 'NULL' + && $column->{Collation} ne $collate) + { + my $name = $column->{Field}; + + print "$table.$name needs to be converted to $charset (collate $collate)...\n"; + + # These will be automatically re-created at the end + # of checksetup. + $self->bz_drop_related_fks($table, $name); + + my $col_info = $self->bz_column_info_real($table, $name); + + # CHANGE COLUMN doesn't take PRIMARY KEY + delete $col_info->{PRIMARYKEY}; + my $sql_def = $self->_bz_schema->get_type_ddl($col_info); + + # We don't want MySQL to actually try to *convert* + # from our current charset to UTF-8, we just want to + # transfer the bytes directly. This is how we do that. + + # The CHARACTER SET part of the definition has to come + # right after the type, which will always come first. + my ($binary, $utf8) = ($sql_def, $sql_def); + my $type = $self->_bz_schema->convert_type($col_info->{TYPE}); + $binary =~ s/(\Q$type\E)/$1 CHARACTER SET binary/; + $utf8 =~ s/(\Q$type\E)/$1 CHARACTER SET $charset COLLATE $collate/; + push(@binary_sql, "MODIFY COLUMN $name $binary"); + push(@utf8_sql, "MODIFY COLUMN $name $utf8"); + } + } # foreach column + + if (@binary_sql) { + my %indexes = %{$self->bz_table_indexes($table)}; + foreach my $index_name (keys %indexes) { + my $index = $indexes{$index_name}; + if ($index->{TYPE} and $index->{TYPE} eq 'FULLTEXT') { + $self->bz_drop_index($table, $index_name); + } + else { + delete $indexes{$index_name}; + } + } + + print "Converting the $table table to UTF-8...\n"; + my $bin + = 'ALTER TABLE ' + . $self->quote_identifier($table) . ' ' + . join(', ', @binary_sql); + my $utf + = 'ALTER TABLE ' + . $self->quote_identifier($table) . ' ' + . join(', ', @utf8_sql, "DEFAULT CHARACTER SET $charset COLLATE $collate"); + $self->do($bin); + $self->do($utf); + + # Re-add any removed FULLTEXT indexes. + foreach my $index (keys %indexes) { + $self->bz_add_index($table, $index, $indexes{$index}); + } + } + else { + $self->do('ALTER TABLE ' + . $self->quote_identifier($table) + . " DEFAULT CHARACTER SET $charset COLLATE $collate"); + } + + } # foreach my $table (@tables) + } + + # Sometimes you can have a situation where all the tables are UTF-8, + # but the database isn't. (This tends to happen when you've done + # a mysqldump.) So we have this change outside of the above block, + # so that it just happens silently if no actual *table* conversion + # needs to happen. + unless ($self->bz_db_is_utf8) { + $self->_alter_db_charset_to_utf8(); + } + + $self->_fix_defaults(); + + # Bug 451735 highlighted a bug in bz_drop_index() which didn't + # check for FKs before trying to delete an index. Consequently, + # the series_creator_idx index was considered to be deleted + # despite it was still present in the DB. That's why we have to + # force the deletion, bypassing the DB schema. + if (!$self->bz_index_info('series', 'series_category_idx')) { + if (!$self->bz_index_info('series', 'series_creator_idx') + && $self->bz_index_info_real('series', 'series_creator_idx')) + { + foreach my $column (qw(creator category subcategory name)) { + $self->bz_drop_related_fks('series', $column); + } + $self->bz_drop_index_raw('series', 'series_creator_idx'); + } + } +} + +sub bz_server_version { + my ($self) = @_; + my $version = $self->SUPER::bz_server_version(); + + warn "VERSION: $version"; + if ($version =~ /MariaDB/) { + (undef, $version) = split(/-/, $version); + } + + return $version; +} + +# When you import a MySQL 3/4 mysqldump into MySQL 5, columns that +# aren't supposed to have defaults will have defaults. This is only +# a minor issue, but it makes our tests fail, and it's good to keep +# the DB actually consistent with what DB::Schema thinks the database +# looks like. So we remove defaults from columns that aren't supposed +# to have them +sub _fix_defaults { + my $self = shift; + my $maj_version = substr($self->bz_server_version, 0, 1); + return if $maj_version < 5; + + # The oldest column that could have this problem is bugs.assigned_to, + # so if it doesn't have the problem, we just skip doing this entirely. + my $assi_def = $self->_bz_raw_column_info('bugs', 'assigned_to'); + my $assi_default = $assi_def->{COLUMN_DEF}; + + # This "ne ''" thing is necessary because _raw_column_info seems to + # return COLUMN_DEF as an empty string for columns that don't have + # a default. + return unless (defined $assi_default && $assi_default ne ''); + + my %fix_columns; + foreach my $table ($self->_bz_real_schema->get_table_list()) { + foreach my $column ($self->bz_table_columns($table)) { + my $abs_def = $self->bz_column_info($table, $column); + + # BLOB/TEXT columns never have defaults + next if $abs_def->{TYPE} =~ /BLOB|TEXT/i; + if (!defined $abs_def->{DEFAULT}) { + + # Get the exact default from the database without any + # "fixing" by bz_column_info_real. + my $raw_info = $self->_bz_raw_column_info($table, $column); + my $raw_default = $raw_info->{COLUMN_DEF}; + if (defined $raw_default) { + if ($raw_default eq '') { + + # Only (var)char columns can have empty strings as + # defaults, so if we got an empty string for some + # other default type, then it's bogus. + next unless $abs_def->{TYPE} =~ /char/i; + $raw_default = "''"; + } + $fix_columns{$table} ||= []; + push(@{$fix_columns{$table}}, $column); + print "$table.$column has incorrect DB default: $raw_default\n"; + } + } + } # foreach $column + } # foreach $table + + print "Fixing defaults...\n"; + foreach my $table (reverse sort keys %fix_columns) { + my @alters = map("ALTER COLUMN $_ DROP DEFAULT", @{$fix_columns{$table}}); + my $sql + = 'ALTER TABLE ' . $self->quote_identifier($table) . ' ' . join(',', @alters); + $self->do($sql); + } +} + +sub utf8_charset { + return 'utf8mb4'; +} + +sub utf8_collate { + return 'utf8mb4_unicode_520_ci'; +} + +sub default_row_format { + my ($class, $table) = @_; + my @no_compress = qw( + bug_user_last_visit + cc + email_rates + logincookies + token_data + tokens + ts_error + ts_exitstatus + ts_funcmap + ts_job + ts_note + user_request_log + votes + ); + return 'Dynamic' if any { $table eq $_ } @no_compress; + return 'Compressed'; +} + +sub _alter_db_charset_to_utf8 { + my $self = shift; + my $db_name = Bugzilla->localconfig->db_name; + my $charset = $self->utf8_charset; + my $collate = $self->utf8_collate; + $self->do("ALTER DATABASE $db_name CHARACTER SET $charset COLLATE $collate"); +} + +sub bz_db_is_utf8 { + my $self = shift; + my $db_charset + = $self->selectrow_arrayref("SHOW VARIABLES LIKE 'character_set_database'"); + + # First column holds the variable name, second column holds the value. + my $charset = $self->utf8_charset; + return $db_charset->[1] eq $charset ? 1 : 0; +} + + +sub bz_enum_initial_values { + my ($self) = @_; + my %enum_values = %{$self->ENUM_DEFAULTS}; + + # Get a complete description of the 'bugs' table; with DBD::MySQL + # there isn't a column-by-column way of doing this. Could use + # $dbh->column_info, but it would go slower and we would have to + # use the undocumented mysql_type_name accessor to get the type + # of each row. + my $sth = $self->prepare("DESCRIBE bugs"); + $sth->execute(); + + # Look for the particular columns we are interested in. + while (my ($thiscol, $thistype) = $sth->fetchrow_array()) { + if (defined $enum_values{$thiscol}) { + + # this is a column of interest. + my @value_list; + if ($thistype and ($thistype =~ /^enum\(/)) { + + # it has an enum type; get the set of values. + while ($thistype =~ /'([^']*)'(.*)/) { + push(@value_list, $1); + $thistype = $2; + } + } + if (@value_list) { + + # record the enum values found. + $enum_values{$thiscol} = \@value_list; + } + } + } + + return \%enum_values; +} + +##################################################################### +# MariaDB-specific Database-Reading Methods +##################################################################### + +=begin private + +=head1 MARIADB-SPECIFIC DATABASE-READING METHODS + +These methods read information about the database from the disk, +instead of from a Schema object. They are only reliable for MariaDB +(see bug 285111 for the reasons why not all DBs use/have functions +like this), but that's OK because we only need them for +backwards-compatibility anyway, for versions of Bugzilla before 2.20. + +=over 4 + +=item C + + Description: Returns an abstract column definition for a column + as it actually exists on disk in the database. + Params: $table - The name of the table the column is on. + $column - The name of the column you want info about. + Returns: An abstract column definition. + If the column does not exist, returns undef. + +=cut + +sub bz_column_info_real { + my ($self, $table, $column) = @_; + my $col_data = $self->_bz_raw_column_info($table, $column); + return $self->_bz_schema->column_info_to_column($col_data); +} + +sub _bz_raw_column_info { + my ($self, $table, $column) = @_; + + # DBD::mysql does not support selecting a specific column, + # so we have to get all the columns on the table and find + # the one we want. + my $info_sth = $self->column_info(undef, undef, $table, '%'); + + # Don't use fetchall_hashref as there's a Win32 DBI bug (292821) + my $col_data; + while ($col_data = $info_sth->fetchrow_hashref) { + last if $col_data->{'COLUMN_NAME'} eq $column; + } + + if (!defined $col_data) { + return undef; + } + return $col_data; +} + +=item C + + Description: Returns information about an index on a table in the database. + Params: $table = name of table containing the index + $index = name of an index + Returns: An abstract index definition, always in hashref format. + If the index does not exist, the function returns undef. + +=cut + +sub bz_index_info_real { + my ($self, $table, $index) = @_; + + my $sth = $self->prepare("SHOW INDEX FROM $table"); + $sth->execute; + + my @fields; + my $index_type; + + # $raw_def will be an arrayref containing the following information: + # 0 = name of the table that the index is on + # 1 = 0 if unique, 1 if not unique + # 2 = name of the index + # 3 = seq_in_index (The order of the current field in the index). + # 4 = Name of ONE column that the index is on + # 5 = 'Collation' of the index. Usually 'A'. + # 6 = Cardinality. Either a number or undef. + # 7 = sub_part. Usually undef. Sometimes 1. + # 8 = "packed". Usually undef. + # 9 = Null. Sometimes undef, sometimes 'YES'. + # 10 = Index_type. The type of the index. Usually either 'BTREE' or 'FULLTEXT' + # 11 = 'Comment.' Usually undef. + while (my $raw_def = $sth->fetchrow_arrayref) { + if ($raw_def->[2] eq $index) { + push(@fields, $raw_def->[4]); + + # No index can be both UNIQUE and FULLTEXT, that's why + # this is written this way. + $index_type = $raw_def->[1] ? '' : 'UNIQUE'; + $index_type = $raw_def->[10] eq 'FULLTEXT' ? 'FULLTEXT' : $index_type; + } + } + + my $retval; + if (scalar(@fields)) { + $retval = {FIELDS => \@fields, TYPE => $index_type}; + } + return $retval; +} + +=item C + + Description: Returns a list of index names on a table in + the database, as it actually exists on disk. + Params: $table - The name of the table you want info about. + Returns: An array of index names. + +=cut + +sub bz_index_list_real { + my ($self, $table) = @_; + my $sth = $self->prepare("SHOW INDEX FROM $table"); + + # Column 3 of a SHOW INDEX statement contains the name of the index. + return @{$self->selectcol_arrayref($sth, {Columns => [3]})}; +} + +##################################################################### +# MariaDB-Specific "Schema Builder" +##################################################################### + +=back + +=head1 MARIADB-SPECIFIC "SCHEMA BUILDER" + +MariaDB needs to be able to read in a legacy database (from before +Schema existed) and create a Schema object out of it. That's what +this code does. + +=end private + +=cut + +# This sub itself is actually written generically, but the subroutines +# that it depends on are database-specific. In particular, the +# bz_column_info_real function would be very difficult to create +# properly for any other DB besides MariaDB. +sub _bz_build_schema_from_disk { + my ($self) = @_; + + my $schema = $self->_bz_schema->get_empty_schema(); + + my @tables = $self->bz_table_list_real(); + if (@tables) { + print "Building Schema object from database...\n"; + } + foreach my $table (@tables) { + $schema->add_table($table); + my @columns = $self->bz_table_columns_real($table); + foreach my $column (@columns) { + my $type_info = $self->bz_column_info_real($table, $column); + $schema->set_column($table, $column, $type_info); + } + + my @indexes = $self->bz_index_list_real($table); + foreach my $index (@indexes) { + unless ($index eq 'PRIMARY') { + my $index_info = $self->bz_index_info_real($table, $index); + ($index_info = $index_info->{FIELDS}) if (!$index_info->{TYPE}); + $schema->set_index($table, $index, $index_info); + } + } + } + + return $schema; +} + +1; diff --git a/Bugzilla/DB/Schema/MariaDB.pm b/Bugzilla/DB/Schema/MariaDB.pm new file mode 100644 index 0000000000..b857d8e5dc --- /dev/null +++ b/Bugzilla/DB/Schema/MariaDB.pm @@ -0,0 +1,463 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. + +package Bugzilla::DB::Schema::MariaDB; + +############################################################################### +# +# DB::Schema implementation for MariaDB +# +############################################################################### + +use 5.10.1; +use strict; +use warnings; + +use Bugzilla::Error; + +use base qw(Bugzilla::DB::Schema); + +# This is for column_info_to_column, to know when a tinyint is a +# boolean and when it's really a tinyint. This only has to be accurate +# up to and through 2.19.3, because that's the only time we need +# column_info_to_column. +# +# This is basically a hash of tables/columns, with one entry for each column +# that should be interpreted as a BOOLEAN instead of as an INT1 when +# reading in the Schema from the disk. The values are discarded; I just +# used "1" for simplicity. +# +# THIS CONSTANT IS ONLY USED FOR UPGRADES FROM 2.18 OR EARLIER. DON'T +# UPDATE IT TO MODERN COLUMN NAMES OR DEFINITIONS. +use constant BOOLEAN_MAP => { + bugs => { + everconfirmed => 1, + reporter_accessible => 1, + cclist_accessible => 1, + qacontact_accessible => 1, + assignee_accessible => 1 + }, + longdescs => {isprivate => 1, already_wrapped => 1}, + attachments => {ispatch => 1, isobsolete => 1, isprivate => 1}, + flags => {is_active => 1}, + flagtypes => { + is_active => 1, + is_requestable => 1, + is_requesteeble => 1, + is_multiplicable => 1 + }, + fielddefs => {mailhead => 1, obsolete => 1}, + bug_type => {isactive => 1}, + bug_status => {isactive => 1}, + resolution => {isactive => 1}, + bug_severity => {isactive => 1}, + priority => {isactive => 1}, + rep_platform => {isactive => 1}, + op_sys => {isactive => 1}, + profiles => {mybugslink => 1, newemailtech => 1}, + namedqueries => {linkinfooter => 1, watchfordiffs => 1}, + groups => {isbuggroup => 1, isactive => 1}, + group_control_map => + {entry => 1, membercontrol => 1, othercontrol => 1, canedit => 1}, + group_group_map => {isbless => 1}, + user_group_map => {isbless => 1, isderived => 1}, + products => {disallownew => 1}, + series => {public => 1}, + whine_queries => {onemailperbug => 1}, + quips => {approved => 1}, + setting => {is_enabled => 1} +}; + +# Maps the db_specific hash backwards, for use in column_info_to_column. +use constant REVERSE_MAPPING => { + + # Boolean and the SERIAL fields are handled in column_info_to_column, + # and so don't have an entry here. + TINYINT => 'INT1', + SMALLINT => 'INT2', + MEDIUMINT => 'INT3', + INTEGER => 'INT4', + + # All the other types have the same name in their abstract version + # as in their db-specific version, so no reverse mapping is needed. +}; + +#------------------------------------------------------------------------------ +sub _initialize { + + my $self = shift; + + $self = $self->SUPER::_initialize(@_); + + $self->{db_specific} = { + + BOOLEAN => 'tinyint', + FALSE => '0', + TRUE => '1', + + INT1 => 'tinyint', + INT2 => 'smallint', + INT3 => 'mediumint', + INT4 => 'integer', + INT5 => 'bigint(20)', + + SMALLSERIAL => 'smallint auto_increment', + MEDIUMSERIAL => 'mediumint auto_increment', + INTSERIAL => 'integer auto_increment', + BIGSERIAL => 'bigint(20) auto_increment', + + TINYTEXT => 'tinytext', + MEDIUMTEXT => 'mediumtext', + LONGTEXT => 'mediumtext', + TEXT => 'text', + + LONGBLOB => 'longblob', + + + NATIVE_DATETIME => 'datetime', + DATETIME => 'timestamp', + DATE => 'date', + }; + + $self->_adjust_schema; + + return $self; + +} #eosub--_initialize + +#------------------------------------------------------------------------------ +sub _get_create_table_ddl { + + # Returns a "create table" SQL statement. + my ($self, $table) = @_; + my $charset = Bugzilla::DB::MariaDB->utf8_charset; + my $collate = Bugzilla::DB::MariaDB->utf8_collate; + my $row_format = Bugzilla::DB::MariaDB->default_row_format($table); + my @parts = ( + $self->SUPER::_get_create_table_ddl($table), 'ENGINE = InnoDB', + "CHARACTER SET $charset COLLATE $collate", "ROW_FORMAT=$row_format", + ); + return join(' ', @parts); +} #eosub--_get_create_table_ddl + +#------------------------------------------------------------------------------ +sub _get_create_index_ddl { + + # Extend superclass method to create FULLTEXT indexes on text fields. + # Returns a "create index" SQL statement. + + my ($self, $table_name, $index_name, $index_fields, $index_type) = @_; + my $dbh = Bugzilla->dbh; + + my $sql = "CREATE "; + $sql .= "$index_type " + if ($index_type eq 'UNIQUE' || $index_type eq 'FULLTEXT'); + $sql + .= "INDEX " + . $dbh->quote_identifier($index_name) . " ON " + . $dbh->quote_identifier($table_name) . " \(" + . join(", ", @$index_fields) . "\)"; + + return ($sql); + +} #eosub--_get_create_index_ddl + +#-------------------------------------------------------------------- + +sub get_create_database_sql { + my ($self, $name) = @_; + + # We only create as UTF-8 if we have no params (meaning we're doing + # a new installation) or if the utf8 param is on. + my $charset = Bugzilla::DB::MariaDB->utf8_charset; + my $collate = Bugzilla::DB::MariaDB->utf8_collate; + return ("CREATE DATABASE $name CHARACTER SET $charset COLLATE $collate"); +} + +# MariaDB has a simpler ALTER TABLE syntax than ANSI. +sub get_alter_column_ddl { + my ($self, $table, $column, $new_def, $set_nulls_to) = @_; + my $old_def = $self->get_column($table, $column); + my %new_def_copy = %$new_def; + if ($old_def->{PRIMARYKEY} && $new_def->{PRIMARYKEY}) { + + # If a column stays a primary key do NOT specify PRIMARY KEY in the + # ALTER TABLE statement. This avoids a MariaDB error that two primary + # keys are not allowed. + delete $new_def_copy{PRIMARYKEY}; + } + + my @statements; + + my $dbh = Bugzilla->dbh; + push(@statements, + "UPDATE " + . $dbh->quote_identifier($table) + . " SET $column = $set_nulls_to WHERE $column IS NULL") + if defined $set_nulls_to; + + # Calling SET DEFAULT or DROP DEFAULT is *way* faster than calling + # CHANGE COLUMN, so just do that if we're just changing the default. + my %old_defaultless = %$old_def; + my %new_defaultless = %$new_def; + delete $old_defaultless{DEFAULT}; + delete $new_defaultless{DEFAULT}; + if (!$self->columns_equal($old_def, $new_def) + && $self->columns_equal(\%new_defaultless, \%old_defaultless)) + { + if (!defined $new_def->{DEFAULT}) { + push(@statements, + "ALTER TABLE " + . $dbh->quote_identifier($table) + . " ALTER COLUMN $column DROP DEFAULT"); + } + else { + push(@statements, + "ALTER TABLE " + . $dbh->quote_identifier($table) + . " ALTER COLUMN $column SET DEFAULT " + . $new_def->{DEFAULT}); + } + } + else { + my $new_ddl = $self->get_type_ddl(\%new_def_copy); + push(@statements, + "ALTER TABLE " + . $dbh->quote_identifier($table) + . " CHANGE COLUMN $column $column $new_ddl"); + } + + if ($old_def->{PRIMARYKEY} && !$new_def->{PRIMARYKEY}) { + + # Dropping a PRIMARY KEY needs an explicit DROP PRIMARY KEY + push(@statements, + 'ALTER TABLE ' . $dbh->quote_identifier($table) . ' DROP PRIMARY KEY'); + } + + return @statements; +} + +sub get_drop_fk_sql { + my ($self, $table, $column, $references) = @_; + my $fk_name = $self->_get_fk_name($table, $column, $references); + my $dbh = Bugzilla->dbh; + my @sql = ( + "ALTER TABLE " . $dbh->quote_identifier($table) . " DROP FOREIGN KEY $fk_name"); + + # MySQL requires, and will create, an index on any column with + # an FK. It will name it after the fk, which we never do. + # So if there's an index named after the fk, we also have to delete it. + if ($dbh->bz_index_info_real($table, $fk_name)) { + push(@sql, $self->get_drop_index_ddl($table, $fk_name)); + } + + return @sql; +} + +sub get_drop_index_ddl { + my ($self, $table, $name) = @_; + return ("DROP INDEX \`$name\` ON $table"); +} + +# A special function for MySQL, for renaming a lot of indexes. +# Index renames is a hash, where the key is a string - the +# old names of the index, and the value is a hash - the index +# definition that we're renaming to, with an extra key of "NAME" +# that contains the new index name. +# The indexes in %indexes must be in hashref format. +sub get_rename_indexes_ddl { + my ($self, $table, %indexes) = @_; + my @keys = keys %indexes or return (); + + my $sql = 'ALTER TABLE' . Bugzilla->dbh->quote_identifier($table) . ' '; + + foreach my $old_name (@keys) { + my $name = $indexes{$old_name}->{NAME}; + my $type = $indexes{$old_name}->{TYPE}; + $type ||= 'INDEX'; + my $fields = join(',', @{$indexes{$old_name}->{FIELDS}}); + + # $old_name needs to be escaped, sometimes, because it was + # a reserved word. + $old_name = '`' . $old_name . '`'; + $sql .= " ADD $type $name ($fields), DROP INDEX $old_name,"; + } + + # Remove the last comma. + chop($sql); + return ($sql); +} + +sub get_set_serial_sql { + my ($self, $table, $column, $value) = @_; + return ("ALTER TABLE " + . Bugzilla->dbh->quote_identifier($table) + . " AUTO_INCREMENT = $value"); +} + +# Converts a DBI column_info output to an abstract column definition. +# Expects to only be called by Bugzilla::DB::MariaDB::_bz_build_schema_from_disk, +# although there's a chance that it will also work properly if called +# elsewhere. +sub column_info_to_column { + my ($self, $column_info) = @_; + + # Unfortunately, we have to break Schema's normal "no database" + # barrier a few times in this function. + my $dbh = Bugzilla->dbh; + + my $table = $column_info->{TABLE_NAME}; + my $col_name = $column_info->{COLUMN_NAME}; + + my $column = {}; + + ($column->{NOTNULL} = 1) if $column_info->{NULLABLE} == 0; + + if ($column_info->{mysql_is_pri_key}) { + + # In MySQL, if a table has no PK, but it has a UNIQUE index, + # that index will show up as the PK. So we have to eliminate + # that possibility. + # Unfortunately, the only way to definitely solve this is + # to break Schema's standard of not touching the live database + # and check if the index called PRIMARY is on that field. + my $pri_index = $dbh->bz_index_info_real($table, 'PRIMARY'); + if ($pri_index && grep($_ eq $col_name, @{$pri_index->{FIELDS}})) { + $column->{PRIMARYKEY} = 1; + } + } + + # MySQL frequently defines a default for a field even when we + # didn't explicitly set one. So we have to have some special + # hacks to determine whether or not we should actually put + # a default in the abstract schema for this field. + if (defined $column_info->{COLUMN_DEF}) { + + # The defaults that MySQL inputs automatically are usually + # something that would be considered "false" by Perl, either + # a 0 or an empty string. (Except for datetime and decimal + # fields, which have their own special auto-defaults.) + # + # Here's how we handle this: If it exists in the schema + # without a default, then we don't use the default. If it + # doesn't exist in the schema, then we're either going to + # be dropping it soon, or it's a custom end-user column, in which + # case having a bogus default won't harm anything. + my $schema_column = $self->get_column($table, $col_name); + unless ( + ( + !$column_info->{COLUMN_DEF} + || $column_info->{COLUMN_DEF} eq '0000-00-00 00:00:00' + || $column_info->{COLUMN_DEF} eq '0.00' + ) + && $schema_column + && !exists $schema_column->{DEFAULT} + ) + { + + my $default = $column_info->{COLUMN_DEF}; + + # Schema uses '0' for the defaults for decimal fields. + $default = 0 if $default =~ /^0\.0+$/; + + # If we're not a number, we're a string and need to be + # quoted. + $default = $dbh->quote($default) if !($default =~ /^(-)?(\d+)(.\d+)?$/); + $column->{DEFAULT} = $default; + } + } + + my $type = $column_info->{TYPE_NAME}; + + # Certain types of columns need the size/precision appended. + if ($type =~ /CHAR$/ || $type eq 'DECIMAL') { + + # This is nicely lowercase and has the size/precision appended. + $type = $column_info->{mysql_type_name}; + } + + # If we're a tinyint, we could be either a BOOLEAN or an INT1. + # Only the BOOLEAN_MAP knows the difference. + elsif ($type eq 'TINYINT' + && exists BOOLEAN_MAP->{$table} + && exists BOOLEAN_MAP->{$table}->{$col_name}) + { + $type = 'BOOLEAN'; + if (exists $column->{DEFAULT}) { + $column->{DEFAULT} = $column->{DEFAULT} ? 'TRUE' : 'FALSE'; + } + } + + # We also need to check if we're an auto_increment field. + elsif ($type =~ /INT/) { + + # Unfortunately, the only way to do this in DBI is to query the + # database, so we have to break the rule here that Schema normally + # doesn't touch the live DB. + my $ref_sth = $dbh->prepare("SELECT $col_name FROM $table LIMIT 1"); + $ref_sth->execute; + if ($ref_sth->{mysql_is_auto_increment}->[0]) { + if ($type eq 'MEDIUMINT') { + $type = 'MEDIUMSERIAL'; + } + elsif ($type eq 'SMALLINT') { + $type = 'SMALLSERIAL'; + } + else { + $type = 'INTSERIAL'; + } + } + $ref_sth->finish; + + } + + # For all other db-specific types, check if they exist in + # REVERSE_MAPPING and use the type found there. + if (exists REVERSE_MAPPING->{$type}) { + $type = REVERSE_MAPPING->{$type}; + } + + $column->{TYPE} = $type; + + #print "$table.$col_name: " . Data::Dumper->Dump([$column]) . "\n"; + + return $column; +} + +sub get_rename_column_ddl { + my ($self, $table, $old_name, $new_name) = @_; + my $def = $self->get_type_ddl($self->get_column($table, $old_name)); + + # MySQL doesn't like having the PRIMARY KEY statement in a rename. + $def =~ s/PRIMARY KEY//i; + return ("ALTER TABLE " + . Bugzilla->dbh->quote_identifier($table) + . " CHANGE COLUMN $old_name $new_name $def"); +} + +sub get_type_ddl { + my $self = shift; + my $type_ddl = $self->SUPER::get_type_ddl(@_); + +# TIMESTAMPS as of 5.6.6 still default to +# 'NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP' +# unless explicitly setup in the table definition. This will change in future releases +# and can be disabled by using 'explicit_defaults_for_timestamp = 1' in my.cnf. +# So instead, we explicitly setup TIMESTAMP types to not be automatic. + if ($type_ddl =~ /^timestamp/i) { + if ($type_ddl !~ /NOT NULL/) { + $type_ddl .= ' NULL DEFAULT NULL'; + } + if ($type_ddl =~ /NOT NULL/ && $type_ddl !~ /DEFAULT/) { + $type_ddl .= ' DEFAULT CURRENT_TIMESTAMP'; + } + } + + return $type_ddl; +} + +1; diff --git a/Bugzilla/DaemonControl.pm b/Bugzilla/DaemonControl.pm index 3909546494..e35c8e4d55 100644 --- a/Bugzilla/DaemonControl.pm +++ b/Bugzilla/DaemonControl.pm @@ -260,7 +260,6 @@ sub assert_database { $dbh = DBI->connect($dsn, $lc->{db_user}, $lc->{db_pass}, $attrs); }; - if ($!) { $assert_dbierrstr = $@; die $@; } $assert_dbierrstr = DBI->errstr() || ''; die "$assert_dbierrstr" if $assert_dbierrstr; Future->wrap($dbh); diff --git a/Bugzilla/Install/Requirements.pm b/Bugzilla/Install/Requirements.pm index 27504e7d85..0527238767 100644 --- a/Bugzilla/Install/Requirements.pm +++ b/Bugzilla/Install/Requirements.pm @@ -265,8 +265,8 @@ sub check_font_file { sub _checking_for { my ($params) = @_; - my ($package, $ok, $wanted, $blacklisted, $found) - = @$params{qw(package ok wanted blacklisted found)}; + my ($package, $ok, $wanted, $blocklisted, $found) + = @$params{qw(package ok wanted blocklisted found)}; my $ok_string = $ok ? install_string('module_ok') : ''; @@ -294,10 +294,10 @@ sub _checking_for { $ok_string = install_string('module_not_found'); } - my $black_string = $blacklisted ? install_string('blacklisted') : ''; + my $block_string = $blocklisted ? install_string('blocklisted') : ''; my $want_string = $wanted ? "$wanted" : install_string('any'); - my $str = sprintf "%s %20s %-11s $ok_string $black_string\n", + my $str = sprintf "%s %20s %-11s $ok_string $block_string\n", (' ' x $checking_for_indent) . install_string('checking_for'), $package, "($want_string)"; print $ok ? $str : colored($str, COLOR_ERROR); diff --git a/Dockerfile b/Dockerfile index 44cca0f828..7ffc09b477 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ ARG BZDB="-mysql8" -FROM bugzilla/bugzilla-perl-slim${BZDB}:20240410.1 +FROM bugzilla/bugzilla-perl-slim${BZDB}:20240419.1 ENV DEBIAN_FRONTEND noninteractive diff --git a/Makefile.PL b/Makefile.PL index 1fd18e7f7b..e5607385a9 100755 --- a/Makefile.PL +++ b/Makefile.PL @@ -239,6 +239,13 @@ my %optional_features = ( prereqs => {runtime => {requires => {'MIME::Parser' => '5.406', 'XML::Twig' => 0}}}, }, + mariadb => { + description => 'MariaDB database support', + prereqs => { + runtime => + {requires => {'DBD::MariaDB' => 0, 'DateTime::Format::MySQL' => '0.06'}} + } + }, mysql => { description => 'MySQL database support', prereqs => { @@ -393,7 +400,7 @@ WriteMakefile( sub MY::postamble { return <<"MAKE"; -GEN_CPANFILE_ARGS = -D better_xff -D mysql -D jsonrpc -D xmlrpc -D documentation -D pg +GEN_CPANFILE_ARGS = -D better_xff -D mariadb -D jsonrpc -D xmlrpc -D documentation -D pg cpanfile: MYMETA.json \t\$(PERLRUN) gen-cpanfile.pl \$(GEN_CPANFILE_ARGS) diff --git a/cpanfile b/cpanfile index 4ec0886cf6..b4c3a0f101 100644 --- a/cpanfile +++ b/cpanfile @@ -8,7 +8,9 @@ requires 'Class::XSAccessor', '1.18'; requires 'Crypt::CBC'; requires 'Crypt::DES'; requires 'Crypt::DES_EDE3'; -requires 'DBD::mysql', '5.003'; +requires 'Crypt::OpenPGP', '1.12'; +requires 'Crypt::SMIME'; +requires 'DBD::MariaDB'; requires 'DBI', '1.614'; requires 'DBIx::Class'; requires 'DBIx::Class::Helpers', '2.034002'; diff --git a/docker-compose.test-mariadb.yml b/docker-compose.test-mariadb.yml new file mode 100644 index 0000000000..832a4594f2 --- /dev/null +++ b/docker-compose.test-mariadb.yml @@ -0,0 +1,68 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +version: '3.6' + +services: + bugzilla6.test: + build: + args: + - BZDB=-mariadb106 + context: . + dockerfile: Dockerfile + command: dev_httpd + tmpfs: + - /tmp + - /run + environment: + - 'BMO_inbound_proxies=*' + - BMO_db_driver=mariadb + - BMO_db_host=bugzilla6.mariadb106 + - BMO_db_name=bugs + - BMO_db_pass=bugs + - BMO_db_user=bugs + - BMO_db_mysql_ssl_get_pubkey=1 + - BMO_memcached_namespace=bugzilla + - BMO_memcached_servers=memcached:11211 + - BMO_ses_username=ses@mozilla.bugs + - BMO_ses_password=password123456789! + - BMO_urlbase=AUTOMATIC + - BUGZILLA_ALLOW_INSECURE_HTTP=1 + - BZ_ANSWERS_FILE=/app/conf/checksetup_answers.txt + - BZ_QA_ANSWERS_FILE=/app/.github/checksetup_answers.txt + - BZ_QA_CONF_FILE=/app/.github/selenium_test.conf + - BZ_QA_CONFIG=1 + - LOCALCONFIG_ENV=1 + - LOG4PERL_CONFIG_FILE=log4perl-test.conf + - LOGGING_PORT=5880 + - PORT=8000 + - TWD_BROWSER=firefox + - TWD_HOST=selenium + - TWD_PORT=4444 + depends_on: + - bugzilla6.mariadb106 + - memcached + - selenium + + bugzilla6.mariadb106: + image: mariadb:10.6 + tmpfs: + - /tmp + logging: + driver: "none" + environment: + - MARIADB_DATABASE=bugs + - MARIADB_ROOT_PASSWORD=bugs + - MARIADB_USER=bugs + - MARIADB_PASSWORD=bugs + - MARIADB_ALLOW_EMPTY_PASSWORD=1 + + memcached: + image: memcached:latest + + selenium: + image: selenium/standalone-firefox:3.141.59 + shm_size: '512m' + #ports: + # - "5900:5900" diff --git a/docker/gen-bugzilla-slim-mariadb106.sh b/docker/gen-bugzilla-slim-mariadb106.sh new file mode 100644 index 0000000000..9cb56c2776 --- /dev/null +++ b/docker/gen-bugzilla-slim-mariadb106.sh @@ -0,0 +1,113 @@ +#!/bin/bash +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This Source Code Form is "Incompatible With Secondary Licenses", as +# defined by the Mozilla Public License, v. 2.0. + +if [ ! -e 'Makefile.PL' ]; then + echo + echo "Please run this from the root of the Bugzilla source tree." + echo + exit -1 +fi +if [ -z "$DOCKER" ]; then + DOCKER=`which docker` +fi +if [ ! -x "$DOCKER" ]; then + echo + echo "You specified a custom Docker executable via the DOCKER" + echo "environment variable at $DOCKER" + echo "which either does not exist or is not executable." + echo "Please fix it to point at a working Docker or remove the" + echo "DOCKER environment variable to use the one in your PATH" + echo "if it exists." + echo + exit -1 +fi +if [ -z "$DOCKER" ]; then + echo + echo "You do not appear to have docker installed or I can't find it." + echo "Windows and Mac versions can be downloaded from" + echo "https://www.docker.com/products/docker-desktop" + echo "Linux users can install using your package manager." + echo + echo "Please install docker or specify the location of the docker" + echo "executable in the DOCKER environment variable and try again." + echo + exit -1 +fi +$DOCKER info 1>/dev/null 2>/dev/null +if [ $? != 0 ]; then + echo + echo "The docker daemon is not running or I can't connect to it." + echo "Please make sure it's running and try again." + echo + exit -1 +fi +if [ ! -f "docker/images/Dockerfile.bugzilla-mariadb106" ]; then + echo + echo "Can't locate the Dockerfile, try running from the root of" + echo "your Bugzilla checkout." + echo + exit -1 +fi + +export DOCKER_CLI_HINTS=false +export CI="" +export CIRCLE_SHA1="" +export CIRCLE_BUILD_URL="" + +# Figure out the tag name to use for the image. We'll do this by generating +# a code based on today's date, then attempt to pull it from DockerHub. If +# we successfully pull, then it already exists, and we bump the interation +# number on the end. +DATE=`date +"%Y%m%d"` +ITER=1 +$DOCKER pull bugzilla/bugzilla-perl-slim-mariadb106:${DATE}.${ITER} >/dev/null 2>/dev/null +while [ $? == 0 ]; do + # as long as we succesfully pull, keep bumping the number on the end + ((ITER++)) + $DOCKER pull bugzilla/bugzilla-perl-slim-mariadb106:${DATE}.${ITER} >/dev/null 2>/dev/null +done +$DOCKER build -t bugzilla/bugzilla-perl-slim-mariadb106:${DATE}.${ITER} -f docker/images/Dockerfile.bugzilla-mariadb106 . +if [ $? == 0 ]; then + echo + echo "The build appears to have succeeded. Don't forget to change the FROM line" + echo "at the top of Dockerfile to use:" + echo " bugzilla/bugzilla-perl-slim-mariadb106:${DATE}.${ITER}" + echo "to make use of this image." + echo + # check if the user is logged in + if [ -z "$PYTHON" ]; then + PYTHON=`which python` + fi + if [ -z "$PYTHON" ]; then + PYTHON=`which python3` + fi + if [ ! -x "$PYTHON" ]; then + echo "The python executable specified in your PYTHON environment value or your PATH is not executable or I can't find it." + exit -1 + fi + AUTHINFO=`$PYTHON -c "import json; print(len(json.load(open('${HOME}/.docker/config.json','r',encoding='utf-8'))['auths']))"` + if [ $AUTHINFO -gt 0 ]; then + # user is logged in + read -p "Do you wish to push to DockerHub? [y/N]: " yesno + case $yesno in + [Yy]*) + echo "Pushing..." + $DOCKER push bugzilla/bugzilla-perl-slim-mariadb106:${DATE}.${ITER} + ;; + *) + echo "Not pushing. You can just run this script again when you're ready" + echo "to push. The prior build result is cached." + ;; + esac + fi +else + echo + echo "Docker build failed. See output above." + echo + exit -1 +fi diff --git a/docker/images/Dockerfile.bugzilla-mariadb106 b/docker/images/Dockerfile.bugzilla-mariadb106 new file mode 100644 index 0000000000..74b5dd5aa1 --- /dev/null +++ b/docker/images/Dockerfile.bugzilla-mariadb106 @@ -0,0 +1,57 @@ +FROM bugzilla/bugzilla-perl-slim:20240410.1 AS builder + +RUN apt-get update \ + && apt-get dist-upgrade -y \ + && apt-get install -y \ + libmariadb-dev \ + apt-file \ + build-essential \ + cmake \ + curl \ + git \ + libcairo-dev \ + libexpat-dev \ + libgd-dev \ + libssl-dev \ + openssl \ + zlib1g-dev \ + unzip \ + wget + +WORKDIR /app + +COPY Makefile.PL Bugzilla.pm gen-cpanfile.pl /app/ +COPY extensions/ /app/extensions/ + +RUN cpanm --notest --quiet App::cpm Module::CPANfile Carton::Snapshot + +RUN perl Makefile.PL +RUN make cpanfile GEN_CPANFILE_ARGS="-D mariadb" + +RUN carton install + +RUN apt-file update +RUN find local -name '*.so' -exec ldd {} \; \ + | egrep -v 'not.found|not.a.dynamic.executable' \ + | awk '$3 {print $3}' \ + | sort -u \ + | xargs -IFILE apt-file search -l FILE \ + | sort -u > PACKAGES + +FROM bugzilla/bugzilla-perl-slim:20240410.1 + +ENV DEBIAN_FRONTEND noninteractive + +COPY --from=builder /app/local /app/local +COPY --from=builder /app/PACKAGES /app/PACKAGES + +RUN apt-get update \ + && apt-get install -y \ + gnupg + +RUN apt-get update \ + && apt-get install -y \ + mariadb-client \ + $(cat /app/PACKAGES) \ + && rm -rf /var/cache/apt/* /var/lib/apt/lists/* + diff --git a/docker/run-tests-in-docker.sh b/docker/run-tests-in-docker.sh index e254998891..592878f413 100644 --- a/docker/run-tests-in-docker.sh +++ b/docker/run-tests-in-docker.sh @@ -56,6 +56,8 @@ if [ "$1" == "pg" ]; then DOCKER_COMPOSE_FILE=docker-compose.test-pg.yml elif [ "$1" == "sqlite" ]; then DOCKER_COMPOSE_FILE=docker-compose.test-sqlite.yml +elif [ "$1" == "mariadb" ]; then + DOCKER_COMPOSE_FILE=docker-compose.test-mariadb.yml fi $DOCKER compose -f $DOCKER_COMPOSE_FILE build if [ $? == 0 ]; then diff --git a/extensions/AntiSpam/Extension.pm b/extensions/AntiSpam/Extension.pm index fbe482bba8..878a17ad65 100644 --- a/extensions/AntiSpam/Extension.pm +++ b/extensions/AntiSpam/Extension.pm @@ -67,7 +67,7 @@ sub _comment_blocking { my $regex = '\b(?:' . join('|', map {quotemeta} @$blocklist) . ')\b'; if ($params->{thetext} =~ /$regex/i) { Bugzilla->audit(sprintf( - "blocked <%s> %s from commenting, blacklisted phrase", + "blocked <%s> %s from commenting, blocklisted phrase", remote_ip(), $user->login )); ThrowUserError('antispam_comment_blocked'); @@ -87,7 +87,7 @@ sub _domain_blocking { undef, $address->host); if ($blocked) { Bugzilla->audit(sprintf( - "blocked <%s> from creating %s, blacklisted domain", + "blocked <%s> from creating %s, blocklisted domain", remote_ip(), $login )); ThrowUserError('account_creation_restricted'); @@ -107,7 +107,7 @@ sub _ip_blocking { undef, $ip); if ($blocked) { Bugzilla->audit( - sprintf("blocked <%s> from creating %s, blacklisted IP", $ip, $login)); + sprintf("blocked <%s> from creating %s, blocklisted IP", $ip, $login)); ThrowUserError('account_creation_restricted'); } } diff --git a/extensions/BzAPI/template/en/default/config.json.tmpl b/extensions/BzAPI/template/en/default/config.json.tmpl index 11474a7aa9..29e528a429 100644 --- a/extensions/BzAPI/template/en/default/config.json.tmpl +++ b/extensions/BzAPI/template/en/default/config.json.tmpl @@ -270,8 +270,8 @@ OLDATTACH2NEW = { "description": "[% (field_descs.${item.name} OR item.description) FILTER json %]", "is_active": [% field.obsolete ? "false" : "true" %], - [% blacklist = ["version", "group", "product", "component"] %] - [% IF ${newname} AND NOT blacklist.contains(newname) %] + [% blocklist = ["version", "group", "product", "component"] %] + [% IF ${newname} AND NOT blocklist.contains(newname) %] "values": [ [% FOREACH value = ${newname} %] "[% value FILTER json %]"[% ',' UNLESS loop.last() %] diff --git a/scripts/cpanfile_fixed_versions.pl b/scripts/cpanfile_fixed_versions.pl index 028382a692..e2ab8018a4 100755 --- a/scripts/cpanfile_fixed_versions.pl +++ b/scripts/cpanfile_fixed_versions.pl @@ -54,8 +54,8 @@ sub _check_vers { # Must do a string comparison as $vnum may be of the form 5.10.1. my $vok = ($vnum ne '-1' && version->new($vnum) >= version->new($wanted)) ? 1 : 0; - if ($vok && $params->{blacklist}) { - $vok = 0 if grep($vnum =~ /$_/, @{$params->{blacklist}}); + if ($vok && $params->{blocklist}) { + $vok = 0 if grep($vnum =~ /$_/, @{$params->{blocklist}}); } return {module => $module, ok => $vok, wanted => $wanted, found => $vnum,}; diff --git a/t/004template.t b/t/004template.t index 61e895bf2f..a044364625 100644 --- a/t/004template.t +++ b/t/004template.t @@ -123,10 +123,10 @@ foreach my $include_path (@include_paths) { # Forbid single quotes to delimit URLs, see bug 926085. if ($data =~ /href=\\?'/) { - ok(0, "$path contains blacklisted constructs: href='...'"); + ok(0, "$path contains blocklisted constructs: href='...'"); } else { - ok(1, "$path contains no blacklisted constructs"); + ok(1, "$path contains no blocklisted constructs"); } } } diff --git a/template/en/default/config.js.tmpl b/template/en/default/config.js.tmpl index f0fe8494bb..c82160993a 100644 --- a/template/en/default/config.js.tmpl +++ b/template/en/default/config.js.tmpl @@ -69,7 +69,7 @@ var [% cf.name FILTER js %] = [ [% FOREACH x = cf.legal_values %]'[% x.name FILT // ======================= // // It is not necessary to list all products and components here. -// Instead, you can define a "blacklist" for some commonly used words +// Instead, you can define a "blocklist" for some commonly used words // or word fragments that occur in a product or component name // but should _not_ trigger product/component search. @@ -89,7 +89,7 @@ var target_milestone = new Object(); // Product and Component Exceptions // ================================ // -// A blacklist for some commonly used words or word fragments +// A blocklist for some commonly used words or word fragments // that occur in a product or component name but should *not* // trigger product/component search in QuickSearch. diff --git a/template/en/default/setup/strings.txt.pl b/template/en/default/setup/strings.txt.pl index 851be09df8..13867aad80 100644 --- a/template/en/default/setup/strings.txt.pl +++ b/template/en/default/setup/strings.txt.pl @@ -25,7 +25,7 @@ problem we ran into, run: ##command## END bad_executable => 'not a valid executable: ##bin##', - blacklisted => '(blacklisted)', + blocklisted => '(blocklisted)', bz_schema_exists_before_220 => <<'END', You are upgrading from a version before 2.20, but the bz_schema table already exists. This means that you restored a mysqldump into the Bugzilla From 11aade46115ba45fc9f893aafdf9520cef99e106 Mon Sep 17 00:00:00 2001 From: Dave Miller Date: Sun, 28 Apr 2024 01:54:14 -0400 Subject: [PATCH 2/3] Bug 1893844: configurable logging with a sensible default (#126) * Adds a localconfig var called "logging_method" which points at a config file in the conf directory (which config file to use, there are several that preconfigure log4perl to log various different ways). * Set the default to syslog. --- Bugzilla/Install/Localconfig.pm | 1 + Bugzilla/Logging.pm | 6 +++++- Bugzilla/Util.pm | 4 ++-- RELEASE_BLOCKERS.md | 2 ++ conf/{log4perl-default.conf => log4perl-file.conf} | 0 conf/log4perl-syslog.conf | 4 ++-- template/en/default/setup/strings.txt.pl | 4 ++++ 7 files changed, 16 insertions(+), 5 deletions(-) rename conf/{log4perl-default.conf => log4perl-file.conf} (100%) diff --git a/Bugzilla/Install/Localconfig.pm b/Bugzilla/Install/Localconfig.pm index 4a19b2c361..ee08842eea 100644 --- a/Bugzilla/Install/Localconfig.pm +++ b/Bugzilla/Install/Localconfig.pm @@ -102,6 +102,7 @@ use constant LOCALCONFIG_VARS => ( {name => 'memcached_namespace', default => "bugzilla:",}, {name => 'urlbase', default => '',}, {name => 'canonical_urlbase', lazy => 1}, + {name => 'logging_method', default => 'syslog'}, {name => 'nobody_user', default => 'nobody@mozilla.org'}, {name => 'attachment_base', default => '',}, {name => 'ses_username', default => '',}, diff --git a/Bugzilla/Logging.pm b/Bugzilla/Logging.pm index 870214207f..9a68e51ee2 100644 --- a/Bugzilla/Logging.pm +++ b/Bugzilla/Logging.pm @@ -14,6 +14,7 @@ use Log::Log4perl qw(:easy); use Log::Log4perl::MDC; use File::Spec::Functions qw(rel2abs catfile); use Bugzilla::Constants qw(bz_locations); +use Bugzilla::Install::Localconfig; use English qw(-no_match_vars $PROGRAM_NAME); sub logfile { @@ -28,7 +29,10 @@ sub fields { } BEGIN { - my $file = $ENV{LOG4PERL_CONFIG_FILE} // 'log4perl-default.conf'; + my $lc = Bugzilla::Install::Localconfig::read_localconfig(); + # Use the environment variable if it exists, otherwise use the value + # from localconfig, or if that's not set, use syslog. + my $file = $ENV{LOG4PERL_CONFIG_FILE} // ("log4perl-" . ($lc->{'logging_method'} // 'syslog') . ".conf"); Log::Log4perl::Logger::create_custom_level('NOTICE', 'WARN', 5, 2); Log::Log4perl->init(rel2abs($file, bz_locations->{confdir})); TRACE("logging enabled in $PROGRAM_NAME"); diff --git a/Bugzilla/Util.pm b/Bugzilla/Util.pm index 14d3311f86..2284cf8cab 100644 --- a/Bugzilla/Util.pm +++ b/Bugzilla/Util.pm @@ -29,7 +29,8 @@ use base qw(Exporter); get_text template_var disable_utf8 enable_utf8 detect_encoding email_filter round extract_nicks); -use Bugzilla::Logging; +# Bugzilla::Logging cannot be used it this file due to circular dependency +#use Bugzilla::Logging; use Bugzilla::Constants; use Bugzilla::RNG qw(irand); @@ -329,7 +330,6 @@ sub do_ssl_redirect_if_required { # If we're already running under SSL, never redirect. return if $ENV{HTTPS} && $ENV{HTTPS} eq 'on'; - DEBUG("Redirect to HTTPS because \$ENV{HTTPS}=$ENV{HTTPS}"); Bugzilla->cgi->redirect_to_https(); } diff --git a/RELEASE_BLOCKERS.md b/RELEASE_BLOCKERS.md index a14c3cb6aa..dbd6b784e9 100644 --- a/RELEASE_BLOCKERS.md +++ b/RELEASE_BLOCKERS.md @@ -58,6 +58,8 @@ app. The defaults need to be updated to log to a more generic location users are likely to have, or walk through setting it during the installation script. +**[COMPLETED]** + # Docker and Containerization I would like the Dockerfile to be rewritten such that the ENTRYPOINT is the diff --git a/conf/log4perl-default.conf b/conf/log4perl-file.conf similarity index 100% rename from conf/log4perl-default.conf rename to conf/log4perl-file.conf diff --git a/conf/log4perl-syslog.conf b/conf/log4perl-syslog.conf index 304887748c..b603ef1651 100644 --- a/conf/log4perl-syslog.conf +++ b/conf/log4perl-syslog.conf @@ -1,8 +1,8 @@ log4perl.rootLogger = INFO, Syslog log4perl.appender.Syslog = Log::Dispatch::Syslog log4perl.appender.Syslog.min_level = notice -log4perl.appender.Syslog.ident = apache -log4perl.appender.Syslog.facility = local4 +log4perl.appender.Syslog.ident = bugzilla +log4perl.appender.Syslog.facility = daemon log4perl.appender.Syslog.logopt = cons,pid log4perl.appender.Syslog.layout = Log::Log4perl::Layout::PatternLayout log4perl.appender.Syslog.layout.ConversionPattern = [%c] %m{chomp}%n diff --git a/template/en/default/setup/strings.txt.pl b/template/en/default/setup/strings.txt.pl index 13867aad80..449015b1e6 100644 --- a/template/en/default/setup/strings.txt.pl +++ b/template/en/default/setup/strings.txt.pl @@ -222,6 +222,10 @@ END If you want to use the "Difference Between Two Patches" feature of the Patch Viewer, please specify the full path to the "interdiff" executable here. +END + localconfig_logging_method => <<'END', +This option specifies the method to use to log any errors or debug messages +create by Bugzilla. This will use the configuration found in conf/log4perl-{logging_method}.conf so anything listed there is valid. Examples are 'syslog', 'docker', 'file', and 'json'. END localconfig_memcached_servers => <<'END', If this option is set, Bugzilla will integrate with Memcached. From b6a4dcc9fdb0774616f59555238eb96587bcb62a Mon Sep 17 00:00:00 2001 From: Dave Miller Date: Sun, 28 Apr 2024 02:50:23 -0400 Subject: [PATCH 3/3] Bug 1433029: .DS_Store should be ignored everywhere not just in root (#122) --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index a89f677c1a..c22c12e759 100644 --- a/.gitignore +++ b/.gitignore @@ -20,7 +20,6 @@ /localconfig.* /index.html /error_reports -/.DS_Store /template_cache /Makefile /MYMETA.* @@ -28,6 +27,7 @@ /blib /Bugzilla-*.*/ /local +.DS_Store .vscode/perl-lang version.json __lbheartbeat__